2024-11-25 12:03:27,944 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-25 12:03:27,959 main DEBUG Took 0.013334 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-25 12:03:27,960 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-25 12:03:27,960 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-25 12:03:27,961 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-25 12:03:27,963 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:03:27,972 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-25 12:03:27,986 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:03:27,988 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:03:27,989 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:03:27,990 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:03:27,991 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:03:27,991 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:03:27,992 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:03:27,992 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:03:27,994 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:03:27,994 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:03:27,995 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:03:27,995 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:03:27,996 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:03:27,997 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:03:27,997 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:03:27,998 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:03:27,998 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:03:27,999 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:03:27,999 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:03:28,000 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:03:28,000 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:03:28,001 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:03:28,002 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:03:28,002 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 12:03:28,003 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:03:28,003 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-25 12:03:28,005 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 12:03:28,007 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-25 12:03:28,010 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-25 12:03:28,010 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-25 12:03:28,012 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-25 12:03:28,012 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-25 12:03:28,025 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-25 12:03:28,028 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-25 12:03:28,031 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-25 12:03:28,032 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-25 12:03:28,032 main DEBUG createAppenders(={Console}) 2024-11-25 12:03:28,033 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-25 12:03:28,034 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-25 12:03:28,034 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-25 12:03:28,035 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-25 12:03:28,035 main DEBUG OutputStream closed 2024-11-25 12:03:28,036 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-25 12:03:28,036 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-25 12:03:28,036 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-25 12:03:28,129 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-25 12:03:28,133 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-25 12:03:28,134 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-25 12:03:28,135 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-25 12:03:28,136 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-25 12:03:28,136 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-25 12:03:28,137 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-25 12:03:28,137 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-25 12:03:28,137 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-25 12:03:28,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-25 12:03:28,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-25 12:03:28,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-25 12:03:28,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-25 12:03:28,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-25 12:03:28,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-25 12:03:28,140 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-25 12:03:28,140 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-25 12:03:28,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-25 12:03:28,143 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25 12:03:28,143 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-25 12:03:28,144 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-25 12:03:28,145 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-25T12:03:28,495 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072 2024-11-25 12:03:28,500 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-25 12:03:28,500 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25T12:03:28,513 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-25T12:03:28,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-25T12:03:28,572 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/cluster_2876dd1b-38a2-aa39-0643-f2db42ed69c6, deleteOnExit=true 2024-11-25T12:03:28,572 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-25T12:03:28,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/test.cache.data in system properties and HBase conf 2024-11-25T12:03:28,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/hadoop.tmp.dir in system properties and HBase conf 2024-11-25T12:03:28,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/hadoop.log.dir in system properties and HBase conf 2024-11-25T12:03:28,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-25T12:03:28,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-25T12:03:28,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-25T12:03:28,701 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-25T12:03:28,852 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T12:03:28,856 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-25T12:03:28,857 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-25T12:03:28,857 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-25T12:03:28,858 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T12:03:28,858 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-25T12:03:28,859 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-25T12:03:28,859 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T12:03:28,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T12:03:28,860 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-25T12:03:28,861 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/nfs.dump.dir in system properties and HBase conf 2024-11-25T12:03:28,861 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/java.io.tmpdir in system properties and HBase conf 2024-11-25T12:03:28,861 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T12:03:28,862 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-25T12:03:28,862 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-25T12:03:29,878 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-25T12:03:29,991 INFO [Time-limited test {}] log.Log(170): Logging initialized @3011ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-25T12:03:30,086 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T12:03:30,177 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T12:03:30,212 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T12:03:30,213 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T12:03:30,215 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-25T12:03:30,231 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T12:03:30,234 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@610d6b27{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/hadoop.log.dir/,AVAILABLE} 2024-11-25T12:03:30,236 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31a6a643{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T12:03:30,518 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7f405bdd{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/java.io.tmpdir/jetty-localhost-38359-hadoop-hdfs-3_4_1-tests_jar-_-any-16466127175085332648/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T12:03:30,530 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5b829fcb{HTTP/1.1, (http/1.1)}{localhost:38359} 2024-11-25T12:03:30,531 INFO [Time-limited test {}] server.Server(415): Started @3552ms 2024-11-25T12:03:31,168 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T12:03:31,177 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T12:03:31,182 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T12:03:31,182 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T12:03:31,183 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-25T12:03:31,185 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@350067e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/hadoop.log.dir/,AVAILABLE} 2024-11-25T12:03:31,186 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@24457f8e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T12:03:31,359 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1e531827{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/java.io.tmpdir/jetty-localhost-45503-hadoop-hdfs-3_4_1-tests_jar-_-any-84222361435383764/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T12:03:31,360 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4ea205b6{HTTP/1.1, (http/1.1)}{localhost:45503} 2024-11-25T12:03:31,360 INFO [Time-limited test {}] server.Server(415): Started @4381ms 2024-11-25T12:03:31,445 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T12:03:32,220 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/cluster_2876dd1b-38a2-aa39-0643-f2db42ed69c6/data/data1/current/BP-261260195-172.17.0.3-1732536209581/current, will proceed with Du for space computation calculation, 2024-11-25T12:03:32,222 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/cluster_2876dd1b-38a2-aa39-0643-f2db42ed69c6/data/data2/current/BP-261260195-172.17.0.3-1732536209581/current, will proceed with Du for space computation calculation, 2024-11-25T12:03:32,326 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T12:03:32,390 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7ea53211f3911f6 with lease ID 0xdb8cd5dc79f39e10: Processing first storage report for DS-e355a87c-d052-4ab8-9e2a-3a6de0c1487f from datanode DatanodeRegistration(127.0.0.1:36647, datanodeUuid=77d49c2a-8a68-48a8-be79-2fca156f8fc0, infoPort=37791, infoSecurePort=0, ipcPort=43741, storageInfo=lv=-57;cid=testClusterID;nsid=9368376;c=1732536209581) 2024-11-25T12:03:32,392 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7ea53211f3911f6 with lease ID 0xdb8cd5dc79f39e10: from storage DS-e355a87c-d052-4ab8-9e2a-3a6de0c1487f node DatanodeRegistration(127.0.0.1:36647, datanodeUuid=77d49c2a-8a68-48a8-be79-2fca156f8fc0, infoPort=37791, infoSecurePort=0, ipcPort=43741, storageInfo=lv=-57;cid=testClusterID;nsid=9368376;c=1732536209581), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-25T12:03:32,392 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7ea53211f3911f6 with lease ID 0xdb8cd5dc79f39e10: Processing first storage report for DS-9a63e444-d463-45bc-9626-f27bff17be90 from datanode DatanodeRegistration(127.0.0.1:36647, datanodeUuid=77d49c2a-8a68-48a8-be79-2fca156f8fc0, infoPort=37791, infoSecurePort=0, ipcPort=43741, storageInfo=lv=-57;cid=testClusterID;nsid=9368376;c=1732536209581) 2024-11-25T12:03:32,393 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7ea53211f3911f6 with lease ID 0xdb8cd5dc79f39e10: from storage DS-9a63e444-d463-45bc-9626-f27bff17be90 node DatanodeRegistration(127.0.0.1:36647, datanodeUuid=77d49c2a-8a68-48a8-be79-2fca156f8fc0, infoPort=37791, infoSecurePort=0, ipcPort=43741, storageInfo=lv=-57;cid=testClusterID;nsid=9368376;c=1732536209581), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T12:03:32,414 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072 2024-11-25T12:03:32,513 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/cluster_2876dd1b-38a2-aa39-0643-f2db42ed69c6/zookeeper_0, clientPort=63609, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/cluster_2876dd1b-38a2-aa39-0643-f2db42ed69c6/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/cluster_2876dd1b-38a2-aa39-0643-f2db42ed69c6/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T12:03:32,536 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63609 2024-11-25T12:03:32,551 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:03:32,557 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:03:32,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741825_1001 (size=7) 2024-11-25T12:03:33,395 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d with version=8 2024-11-25T12:03:33,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/hbase-staging 2024-11-25T12:03:33,531 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-25T12:03:33,899 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2bf237fd86ca:0 server-side Connection retries=45 2024-11-25T12:03:33,919 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:03:33,919 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T12:03:33,926 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T12:03:33,927 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:03:33,927 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T12:03:34,169 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T12:03:34,259 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-25T12:03:34,272 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-25T12:03:34,277 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T12:03:34,319 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 13668 (auto-detected) 2024-11-25T12:03:34,320 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-25T12:03:34,358 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40019 2024-11-25T12:03:34,395 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40019 connecting to ZooKeeper ensemble=127.0.0.1:63609 2024-11-25T12:03:34,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:400190x0, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T12:03:34,450 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40019-0x10076d2ae840000 connected 2024-11-25T12:03:34,488 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:03:34,492 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:03:34,505 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T12:03:34,510 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d, hbase.cluster.distributed=false 2024-11-25T12:03:34,549 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T12:03:34,557 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40019 2024-11-25T12:03:34,558 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40019 2024-11-25T12:03:34,558 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40019 2024-11-25T12:03:34,559 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40019 2024-11-25T12:03:34,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40019 2024-11-25T12:03:34,731 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2bf237fd86ca:0 server-side Connection retries=45 2024-11-25T12:03:34,733 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:03:34,734 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T12:03:34,734 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T12:03:34,734 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T12:03:34,735 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T12:03:34,738 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T12:03:34,742 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T12:03:34,743 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37643 2024-11-25T12:03:34,746 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37643 connecting to ZooKeeper ensemble=127.0.0.1:63609 2024-11-25T12:03:34,748 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:03:34,753 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:03:34,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:376430x0, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T12:03:34,764 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37643-0x10076d2ae840001 connected 2024-11-25T12:03:34,764 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T12:03:34,769 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T12:03:34,780 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T12:03:34,784 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T12:03:34,792 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T12:03:34,793 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37643 2024-11-25T12:03:34,793 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37643 2024-11-25T12:03:34,794 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37643 2024-11-25T12:03:34,795 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37643 2024-11-25T12:03:34,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37643 2024-11-25T12:03:34,820 DEBUG [M:0;2bf237fd86ca:40019 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2bf237fd86ca:40019 2024-11-25T12:03:34,821 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2bf237fd86ca,40019,1732536213609 2024-11-25T12:03:34,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:03:34,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:03:34,834 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2bf237fd86ca,40019,1732536213609 2024-11-25T12:03:34,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:03:34,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T12:03:34,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:03:34,866 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T12:03:34,867 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2bf237fd86ca,40019,1732536213609 from backup master directory 2024-11-25T12:03:34,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2bf237fd86ca,40019,1732536213609 2024-11-25T12:03:34,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:03:34,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T12:03:34,872 WARN [master/2bf237fd86ca:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T12:03:34,872 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2bf237fd86ca,40019,1732536213609 2024-11-25T12:03:34,875 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-25T12:03:34,877 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-25T12:03:34,960 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/hbase.id] with ID: 99ca1eae-ba4d-457b-be97-75f5434d5de3 2024-11-25T12:03:34,961 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/.tmp/hbase.id 2024-11-25T12:03:34,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741826_1002 (size=42) 2024-11-25T12:03:35,377 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/.tmp/hbase.id]:[hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/hbase.id] 2024-11-25T12:03:35,433 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T12:03:35,439 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T12:03:35,462 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-11-25T12:03:35,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:03:35,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:03:35,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741827_1003 (size=196) 2024-11-25T12:03:35,907 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:03:35,911 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T12:03:35,929 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:35,934 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:03:35,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741828_1004 (size=1189) 2024-11-25T12:03:36,409 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/data/master/store 2024-11-25T12:03:36,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741829_1005 (size=34) 2024-11-25T12:03:36,839 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-25T12:03:36,842 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:03:36,844 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T12:03:36,844 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:03:36,844 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:03:36,846 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T12:03:36,846 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:03:36,846 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:03:36,848 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732536216843Disabling compacts and flushes for region at 1732536216843Disabling writes for close at 1732536216846 (+3 ms)Writing region close event to WAL at 1732536216846Closed at 1732536216846 2024-11-25T12:03:36,850 WARN [master/2bf237fd86ca:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/data/master/store/.initializing 2024-11-25T12:03:36,851 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/WALs/2bf237fd86ca,40019,1732536213609 2024-11-25T12:03:36,860 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:03:36,878 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2bf237fd86ca%2C40019%2C1732536213609, suffix=, logDir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/WALs/2bf237fd86ca,40019,1732536213609, archiveDir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/oldWALs, maxLogs=10 2024-11-25T12:03:36,913 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/WALs/2bf237fd86ca,40019,1732536213609/2bf237fd86ca%2C40019%2C1732536213609.1732536216885, exclude list is [], retry=0 2024-11-25T12:03:36,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36647,DS-e355a87c-d052-4ab8-9e2a-3a6de0c1487f,DISK] 2024-11-25T12:03:36,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-25T12:03:36,979 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/WALs/2bf237fd86ca,40019,1732536213609/2bf237fd86ca%2C40019%2C1732536213609.1732536216885 2024-11-25T12:03:36,980 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37791:37791)] 2024-11-25T12:03:36,981 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:03:36,982 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:03:36,986 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:03:36,987 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:03:37,039 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:03:37,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T12:03:37,075 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:03:37,078 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:03:37,078 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:03:37,082 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T12:03:37,082 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:03:37,084 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:03:37,084 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:03:37,087 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T12:03:37,087 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:03:37,088 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:03:37,089 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:03:37,091 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T12:03:37,092 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:03:37,093 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:03:37,093 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:03:37,098 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:03:37,100 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:03:37,107 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:03:37,108 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:03:37,113 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T12:03:37,116 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T12:03:37,123 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:03:37,125 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67819946, jitterRate=0.010595947504043579}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T12:03:37,130 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732536217004Initializing all the Stores at 1732536217007 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732536217008 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536217009 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536217009Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536217009Cleaning up temporary data from old regions at 1732536217108 (+99 ms)Region opened successfully at 1732536217130 (+22 ms) 2024-11-25T12:03:37,132 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T12:03:37,168 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bd680fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2bf237fd86ca/172.17.0.3:0 2024-11-25T12:03:37,202 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T12:03:37,214 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T12:03:37,214 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T12:03:37,217 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T12:03:37,219 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-25T12:03:37,224 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-25T12:03:37,224 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T12:03:37,253 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T12:03:37,262 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T12:03:37,264 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T12:03:37,267 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T12:03:37,268 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T12:03:37,270 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T12:03:37,273 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T12:03:37,276 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T12:03:37,278 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T12:03:37,279 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T12:03:37,282 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T12:03:37,303 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T12:03:37,305 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T12:03:37,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T12:03:37,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T12:03:37,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:03:37,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:03:37,314 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2bf237fd86ca,40019,1732536213609, sessionid=0x10076d2ae840000, setting cluster-up flag (Was=false) 2024-11-25T12:03:37,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:03:37,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:03:37,338 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T12:03:37,341 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2bf237fd86ca,40019,1732536213609 2024-11-25T12:03:37,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:03:37,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:03:37,354 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T12:03:37,357 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2bf237fd86ca,40019,1732536213609 2024-11-25T12:03:37,365 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T12:03:37,407 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(746): ClusterId : 99ca1eae-ba4d-457b-be97-75f5434d5de3 2024-11-25T12:03:37,411 DEBUG [RS:0;2bf237fd86ca:37643 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T12:03:37,421 DEBUG [RS:0;2bf237fd86ca:37643 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T12:03:37,421 DEBUG [RS:0;2bf237fd86ca:37643 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T12:03:37,425 DEBUG [RS:0;2bf237fd86ca:37643 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T12:03:37,426 DEBUG [RS:0;2bf237fd86ca:37643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0df73d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2bf237fd86ca/172.17.0.3:0 2024-11-25T12:03:37,446 DEBUG [RS:0;2bf237fd86ca:37643 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2bf237fd86ca:37643 2024-11-25T12:03:37,450 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T12:03:37,450 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T12:03:37,450 DEBUG [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T12:03:37,454 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(2659): reportForDuty to master=2bf237fd86ca,40019,1732536213609 with port=37643, startcode=1732536214674 2024-11-25T12:03:37,468 DEBUG [RS:0;2bf237fd86ca:37643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T12:03:37,475 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T12:03:37,487 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T12:03:37,496 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T12:03:37,503 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2bf237fd86ca,40019,1732536213609 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T12:03:37,513 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55343, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T12:03:37,514 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2bf237fd86ca:0, corePoolSize=5, maxPoolSize=5 2024-11-25T12:03:37,514 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2bf237fd86ca:0, corePoolSize=5, maxPoolSize=5 2024-11-25T12:03:37,514 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2bf237fd86ca:0, corePoolSize=5, maxPoolSize=5 2024-11-25T12:03:37,514 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2bf237fd86ca:0, corePoolSize=5, maxPoolSize=5 2024-11-25T12:03:37,515 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2bf237fd86ca:0, corePoolSize=10, maxPoolSize=10 2024-11-25T12:03:37,515 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2bf237fd86ca:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:03:37,515 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2bf237fd86ca:0, corePoolSize=2, maxPoolSize=2 2024-11-25T12:03:37,516 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2bf237fd86ca:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:03:37,522 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40019 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:37,532 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T12:03:37,533 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T12:03:37,535 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732536247535 2024-11-25T12:03:37,537 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T12:03:37,538 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T12:03:37,545 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T12:03:37,546 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T12:03:37,546 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T12:03:37,546 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T12:03:37,550 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:03:37,548 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:37,550 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T12:03:37,556 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T12:03:37,558 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T12:03:37,558 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T12:03:37,561 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T12:03:37,562 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T12:03:37,564 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2bf237fd86ca:0:becomeActiveMaster-HFileCleaner.large.0-1732536217563,5,FailOnTimeoutGroup] 2024-11-25T12:03:37,569 DEBUG [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-25T12:03:37,569 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2bf237fd86ca:0:becomeActiveMaster-HFileCleaner.small.0-1732536217564,5,FailOnTimeoutGroup] 2024-11-25T12:03:37,569 WARN [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-25T12:03:37,569 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:37,569 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T12:03:37,570 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:37,571 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:37,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741831_1007 (size=1321) 2024-11-25T12:03:37,671 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(2659): reportForDuty to master=2bf237fd86ca,40019,1732536213609 with port=37643, startcode=1732536214674 2024-11-25T12:03:37,678 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40019 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:37,681 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40019 {}] master.ServerManager(517): Registering regionserver=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:37,693 DEBUG [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d 2024-11-25T12:03:37,693 DEBUG [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33743 2024-11-25T12:03:37,693 DEBUG [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T12:03:37,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T12:03:37,700 DEBUG [RS:0;2bf237fd86ca:37643 {}] zookeeper.ZKUtil(111): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:37,700 WARN [RS:0;2bf237fd86ca:37643 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T12:03:37,701 INFO [RS:0;2bf237fd86ca:37643 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:03:37,701 DEBUG [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/WALs/2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:37,716 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2bf237fd86ca,37643,1732536214674] 2024-11-25T12:03:37,749 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T12:03:37,778 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T12:03:37,785 INFO [RS:0;2bf237fd86ca:37643 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T12:03:37,785 INFO [RS:0;2bf237fd86ca:37643 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:37,787 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T12:03:37,794 INFO [RS:0;2bf237fd86ca:37643 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T12:03:37,796 INFO [RS:0;2bf237fd86ca:37643 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:37,796 DEBUG [RS:0;2bf237fd86ca:37643 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2bf237fd86ca:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:03:37,796 DEBUG [RS:0;2bf237fd86ca:37643 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2bf237fd86ca:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:03:37,797 DEBUG [RS:0;2bf237fd86ca:37643 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2bf237fd86ca:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:03:37,797 DEBUG [RS:0;2bf237fd86ca:37643 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2bf237fd86ca:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:03:37,797 DEBUG [RS:0;2bf237fd86ca:37643 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2bf237fd86ca:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:03:37,797 DEBUG [RS:0;2bf237fd86ca:37643 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2bf237fd86ca:0, corePoolSize=2, maxPoolSize=2 2024-11-25T12:03:37,798 DEBUG [RS:0;2bf237fd86ca:37643 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2bf237fd86ca:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:03:37,798 DEBUG [RS:0;2bf237fd86ca:37643 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2bf237fd86ca:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:03:37,798 DEBUG [RS:0;2bf237fd86ca:37643 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2bf237fd86ca:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:03:37,798 DEBUG [RS:0;2bf237fd86ca:37643 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2bf237fd86ca:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:03:37,799 DEBUG [RS:0;2bf237fd86ca:37643 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2bf237fd86ca:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:03:37,799 DEBUG [RS:0;2bf237fd86ca:37643 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2bf237fd86ca:0, corePoolSize=1, maxPoolSize=1 2024-11-25T12:03:37,799 DEBUG [RS:0;2bf237fd86ca:37643 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2bf237fd86ca:0, corePoolSize=3, maxPoolSize=3 2024-11-25T12:03:37,799 DEBUG [RS:0;2bf237fd86ca:37643 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0, corePoolSize=3, maxPoolSize=3 2024-11-25T12:03:37,801 INFO [RS:0;2bf237fd86ca:37643 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:37,801 INFO [RS:0;2bf237fd86ca:37643 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:37,802 INFO [RS:0;2bf237fd86ca:37643 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:37,802 INFO [RS:0;2bf237fd86ca:37643 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:37,802 INFO [RS:0;2bf237fd86ca:37643 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:37,802 INFO [RS:0;2bf237fd86ca:37643 {}] hbase.ChoreService(168): Chore ScheduledChore name=2bf237fd86ca,37643,1732536214674-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T12:03:37,840 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T12:03:37,843 INFO [RS:0;2bf237fd86ca:37643 {}] hbase.ChoreService(168): Chore ScheduledChore name=2bf237fd86ca,37643,1732536214674-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:37,843 INFO [RS:0;2bf237fd86ca:37643 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:37,843 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.Replication(171): 2bf237fd86ca,37643,1732536214674 started 2024-11-25T12:03:37,869 INFO [RS:0;2bf237fd86ca:37643 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:37,869 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(1482): Serving as 2bf237fd86ca,37643,1732536214674, RpcServer on 2bf237fd86ca/172.17.0.3:37643, sessionid=0x10076d2ae840001 2024-11-25T12:03:37,871 DEBUG [RS:0;2bf237fd86ca:37643 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T12:03:37,871 DEBUG [RS:0;2bf237fd86ca:37643 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:37,871 DEBUG [RS:0;2bf237fd86ca:37643 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2bf237fd86ca,37643,1732536214674' 2024-11-25T12:03:37,871 DEBUG [RS:0;2bf237fd86ca:37643 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T12:03:37,872 DEBUG [RS:0;2bf237fd86ca:37643 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T12:03:37,873 DEBUG [RS:0;2bf237fd86ca:37643 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T12:03:37,873 DEBUG [RS:0;2bf237fd86ca:37643 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T12:03:37,874 DEBUG [RS:0;2bf237fd86ca:37643 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:37,874 DEBUG [RS:0;2bf237fd86ca:37643 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2bf237fd86ca,37643,1732536214674' 2024-11-25T12:03:37,874 DEBUG [RS:0;2bf237fd86ca:37643 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T12:03:37,875 DEBUG [RS:0;2bf237fd86ca:37643 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T12:03:37,876 DEBUG [RS:0;2bf237fd86ca:37643 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T12:03:37,876 INFO [RS:0;2bf237fd86ca:37643 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T12:03:37,876 INFO [RS:0;2bf237fd86ca:37643 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T12:03:37,975 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T12:03:37,976 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d 2024-11-25T12:03:37,984 INFO [RS:0;2bf237fd86ca:37643 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T12:03:37,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741832_1008 (size=32) 2024-11-25T12:03:37,989 INFO [RS:0;2bf237fd86ca:37643 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2bf237fd86ca%2C37643%2C1732536214674, suffix=, logDir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/WALs/2bf237fd86ca,37643,1732536214674, archiveDir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/oldWALs, maxLogs=32 2024-11-25T12:03:37,991 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:03:37,994 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T12:03:37,998 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T12:03:37,998 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:03:37,999 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:03:38,000 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T12:03:38,003 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T12:03:38,003 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:03:38,004 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:03:38,004 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T12:03:38,008 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T12:03:38,008 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:03:38,009 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:03:38,009 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T12:03:38,012 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T12:03:38,012 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:03:38,012 DEBUG [RS:0;2bf237fd86ca:37643 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/WALs/2bf237fd86ca,37643,1732536214674/2bf237fd86ca%2C37643%2C1732536214674.1732536217992, exclude list is [], retry=0 2024-11-25T12:03:38,013 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:03:38,014 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T12:03:38,016 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740 2024-11-25T12:03:38,017 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740 2024-11-25T12:03:38,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36647,DS-e355a87c-d052-4ab8-9e2a-3a6de0c1487f,DISK] 2024-11-25T12:03:38,022 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T12:03:38,023 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T12:03:38,024 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:03:38,024 INFO [RS:0;2bf237fd86ca:37643 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/WALs/2bf237fd86ca,37643,1732536214674/2bf237fd86ca%2C37643%2C1732536214674.1732536217992 2024-11-25T12:03:38,027 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T12:03:38,029 DEBUG [RS:0;2bf237fd86ca:37643 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37791:37791)] 2024-11-25T12:03:38,034 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:03:38,035 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63134112, jitterRate=-0.05922842025756836}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:03:38,038 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732536217991Initializing all the Stores at 1732536217993 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732536217993Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732536217994 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536217994Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732536217994Cleaning up temporary data from old regions at 1732536218023 (+29 ms)Region opened successfully at 1732536218038 (+15 ms) 2024-11-25T12:03:38,038 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T12:03:38,038 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T12:03:38,038 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T12:03:38,038 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T12:03:38,039 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T12:03:38,041 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T12:03:38,041 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732536218038Disabling compacts and flushes for region at 1732536218038Disabling writes for close at 1732536218038Writing region close event to WAL at 1732536218040 (+2 ms)Closed at 1732536218041 (+1 ms) 2024-11-25T12:03:38,045 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T12:03:38,045 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T12:03:38,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T12:03:38,066 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T12:03:38,069 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T12:03:38,223 DEBUG [2bf237fd86ca:40019 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-25T12:03:38,239 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:38,248 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2bf237fd86ca,37643,1732536214674, state=OPENING 2024-11-25T12:03:38,254 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T12:03:38,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:03:38,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:03:38,258 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:03:38,259 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:03:38,261 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T12:03:38,264 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2bf237fd86ca,37643,1732536214674}] 2024-11-25T12:03:38,444 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T12:03:38,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58659, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T12:03:38,462 INFO [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T12:03:38,463 INFO [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T12:03:38,465 INFO [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T12:03:38,469 INFO [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2bf237fd86ca%2C37643%2C1732536214674.meta, suffix=.meta, logDir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/WALs/2bf237fd86ca,37643,1732536214674, archiveDir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/oldWALs, maxLogs=32 2024-11-25T12:03:38,496 DEBUG [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/WALs/2bf237fd86ca,37643,1732536214674/2bf237fd86ca%2C37643%2C1732536214674.meta.1732536218472.meta, exclude list is [], retry=0 2024-11-25T12:03:38,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36647,DS-e355a87c-d052-4ab8-9e2a-3a6de0c1487f,DISK] 2024-11-25T12:03:38,513 INFO [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/WALs/2bf237fd86ca,37643,1732536214674/2bf237fd86ca%2C37643%2C1732536214674.meta.1732536218472.meta 2024-11-25T12:03:38,521 DEBUG [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37791:37791)] 2024-11-25T12:03:38,521 DEBUG [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:03:38,524 DEBUG [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T12:03:38,528 DEBUG [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T12:03:38,536 INFO [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T12:03:38,542 DEBUG [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T12:03:38,543 DEBUG [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:03:38,543 DEBUG [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T12:03:38,543 DEBUG [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T12:03:38,549 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T12:03:38,552 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T12:03:38,552 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:03:38,553 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:03:38,554 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T12:03:38,556 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T12:03:38,556 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:03:38,557 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:03:38,557 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T12:03:38,559 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T12:03:38,559 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:03:38,560 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:03:38,560 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T12:03:38,562 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T12:03:38,562 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:03:38,563 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T12:03:38,564 DEBUG [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T12:03:38,566 DEBUG [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740 2024-11-25T12:03:38,571 DEBUG [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740 2024-11-25T12:03:38,574 DEBUG [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T12:03:38,574 DEBUG [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T12:03:38,575 DEBUG [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:03:38,578 DEBUG [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T12:03:38,580 INFO [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68354183, jitterRate=0.01855669915676117}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:03:38,580 DEBUG [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T12:03:38,583 DEBUG [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732536218544Writing region info on filesystem at 1732536218544Initializing all the Stores at 1732536218547 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732536218547Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732536218549 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536218549Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732536218549Cleaning up temporary data from old regions at 1732536218574 (+25 ms)Running coprocessor post-open hooks at 1732536218581 (+7 ms)Region opened successfully at 1732536218582 (+1 ms) 2024-11-25T12:03:38,594 INFO [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732536218434 2024-11-25T12:03:38,607 DEBUG [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T12:03:38,608 INFO [RS_OPEN_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T12:03:38,610 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:38,612 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2bf237fd86ca,37643,1732536214674, state=OPEN 2024-11-25T12:03:38,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T12:03:38,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T12:03:38,618 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:03:38,618 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T12:03:38,619 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:38,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T12:03:38,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2bf237fd86ca,37643,1732536214674 in 356 msec 2024-11-25T12:03:38,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T12:03:38,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 573 msec 2024-11-25T12:03:38,635 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T12:03:38,635 INFO [PEWorker-3 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T12:03:38,661 DEBUG [PEWorker-3 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:03:38,662 DEBUG [PEWorker-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:03:38,690 DEBUG [PEWorker-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:03:38,693 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59953, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:03:38,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3150 sec 2024-11-25T12:03:38,719 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732536218719, completionTime=-1 2024-11-25T12:03:38,723 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-25T12:03:38,723 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T12:03:38,798 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-25T12:03:38,798 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732536278798 2024-11-25T12:03:38,798 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732536338798 2024-11-25T12:03:38,798 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 75 msec 2024-11-25T12:03:38,801 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2bf237fd86ca,40019,1732536213609-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:38,801 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2bf237fd86ca,40019,1732536213609-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:38,801 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2bf237fd86ca,40019,1732536213609-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:38,803 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2bf237fd86ca:40019, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:38,804 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:38,804 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:38,813 DEBUG [master/2bf237fd86ca:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T12:03:38,839 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.967sec 2024-11-25T12:03:38,841 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T12:03:38,842 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T12:03:38,843 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T12:03:38,844 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T12:03:38,844 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T12:03:38,845 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2bf237fd86ca,40019,1732536213609-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T12:03:38,845 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2bf237fd86ca,40019,1732536213609-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T12:03:38,855 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T12:03:38,856 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T12:03:38,857 INFO [master/2bf237fd86ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2bf237fd86ca,40019,1732536213609-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T12:03:38,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e70f62a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:38,917 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-25T12:03:38,917 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-25T12:03:38,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:03:38,924 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:03:38,933 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:03:38,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:03:38,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:03:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3545a7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:38,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:03:38,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:03:38,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:03:38,947 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:03:38,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cb14d3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:38,952 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:03:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:03:38,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:03:38,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:03:38,994 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2bf237fd86ca,40019,1732536213609 2024-11-25T12:03:39,013 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=900, ProcessCount=11, AvailableMemoryMB=5827 2024-11-25T12:03:39,024 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T12:03:39,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 2bf237fd86ca,40019,1732536213609 2024-11-25T12:03:39,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5164c17f 2024-11-25T12:03:39,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T12:03:39,047 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T12:03:39,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T12:03:39,056 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:03:39,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T12:03:39,071 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T12:03:39,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-25T12:03:39,073 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:03:39,076 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T12:03:39,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T12:03:39,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741835_1011 (size=959) 2024-11-25T12:03:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T12:03:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T12:03:39,494 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d 2024-11-25T12:03:39,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741836_1012 (size=53) 2024-11-25T12:03:39,505 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:03:39,505 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 057d2058e8441af178fff3bc63c12b69, disabling compactions & flushes 2024-11-25T12:03:39,505 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:39,506 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:39,506 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. after waiting 0 ms 2024-11-25T12:03:39,506 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:39,506 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:39,506 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 057d2058e8441af178fff3bc63c12b69: Waiting for close lock at 1732536219505Disabling compacts and flushes for region at 1732536219505Disabling writes for close at 1732536219506 (+1 ms)Writing region close event to WAL at 1732536219506Closed at 1732536219506 2024-11-25T12:03:39,508 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T12:03:39,514 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732536219508"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732536219508"}]},"ts":"1732536219508"} 2024-11-25T12:03:39,520 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T12:03:39,522 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T12:03:39,525 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536219522"}]},"ts":"1732536219522"} 2024-11-25T12:03:39,530 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T12:03:39,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=057d2058e8441af178fff3bc63c12b69, ASSIGN}] 2024-11-25T12:03:39,535 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=057d2058e8441af178fff3bc63c12b69, ASSIGN 2024-11-25T12:03:39,537 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=057d2058e8441af178fff3bc63c12b69, ASSIGN; state=OFFLINE, location=2bf237fd86ca,37643,1732536214674; forceNewPlan=false, retain=false 2024-11-25T12:03:39,688 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=057d2058e8441af178fff3bc63c12b69, regionState=OPENING, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:39,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=057d2058e8441af178fff3bc63c12b69, ASSIGN because future has completed 2024-11-25T12:03:39,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674}] 2024-11-25T12:03:39,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T12:03:39,857 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:39,858 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:03:39,858 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:39,858 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:03:39,858 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:39,858 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:39,861 INFO [StoreOpener-057d2058e8441af178fff3bc63c12b69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:39,864 INFO [StoreOpener-057d2058e8441af178fff3bc63c12b69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 057d2058e8441af178fff3bc63c12b69 columnFamilyName A 2024-11-25T12:03:39,864 DEBUG [StoreOpener-057d2058e8441af178fff3bc63c12b69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:03:39,865 INFO [StoreOpener-057d2058e8441af178fff3bc63c12b69-1 {}] regionserver.HStore(327): Store=057d2058e8441af178fff3bc63c12b69/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:03:39,866 INFO [StoreOpener-057d2058e8441af178fff3bc63c12b69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:39,870 INFO [StoreOpener-057d2058e8441af178fff3bc63c12b69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 057d2058e8441af178fff3bc63c12b69 columnFamilyName B 2024-11-25T12:03:39,871 DEBUG [StoreOpener-057d2058e8441af178fff3bc63c12b69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:03:39,872 INFO [StoreOpener-057d2058e8441af178fff3bc63c12b69-1 {}] regionserver.HStore(327): Store=057d2058e8441af178fff3bc63c12b69/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:03:39,872 INFO [StoreOpener-057d2058e8441af178fff3bc63c12b69-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:39,875 INFO [StoreOpener-057d2058e8441af178fff3bc63c12b69-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 057d2058e8441af178fff3bc63c12b69 columnFamilyName C 2024-11-25T12:03:39,875 DEBUG [StoreOpener-057d2058e8441af178fff3bc63c12b69-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:03:39,876 INFO [StoreOpener-057d2058e8441af178fff3bc63c12b69-1 {}] regionserver.HStore(327): Store=057d2058e8441af178fff3bc63c12b69/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:03:39,876 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:39,878 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:39,879 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:39,881 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:39,882 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:39,884 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:03:39,889 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:39,903 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:03:39,905 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 057d2058e8441af178fff3bc63c12b69; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69807366, jitterRate=0.04021081328392029}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:03:39,905 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:39,906 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 057d2058e8441af178fff3bc63c12b69: Running coprocessor pre-open hook at 1732536219859Writing region info on filesystem at 1732536219859Initializing all the Stores at 1732536219861 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536219861Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536219861Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536219861Cleaning up temporary data from old regions at 1732536219882 (+21 ms)Running coprocessor post-open hooks at 1732536219905 (+23 ms)Region opened successfully at 1732536219906 (+1 ms) 2024-11-25T12:03:39,909 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., pid=6, masterSystemTime=1732536219850 2024-11-25T12:03:39,914 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:39,915 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:39,915 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=057d2058e8441af178fff3bc63c12b69, regionState=OPEN, openSeqNum=2, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:39,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 because future has completed 2024-11-25T12:03:39,930 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T12:03:39,930 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 in 231 msec 2024-11-25T12:03:39,936 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T12:03:39,936 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=057d2058e8441af178fff3bc63c12b69, ASSIGN in 398 msec 2024-11-25T12:03:39,938 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T12:03:39,938 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536219938"}]},"ts":"1732536219938"} 2024-11-25T12:03:39,942 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T12:03:39,944 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T12:03:39,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 884 msec 2024-11-25T12:03:40,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T12:03:40,218 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:03:40,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dfbcf24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,226 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:03:40,226 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:03:40,229 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:03:40,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:03:40,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:03:40,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ecb3032, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:03:40,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:03:40,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:03:40,234 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:03:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3922c1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,237 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:03:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:03:40,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:03:40,243 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:03:40,246 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T12:03:40,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 2bf237fd86ca,40019,1732536213609 2024-11-25T12:03:40,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7eef39e4 2024-11-25T12:03:40,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T12:03:40,259 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T12:03:40,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175b2636, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,265 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:03:40,265 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:03:40,270 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:03:40,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:03:40,271 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:03:40,271 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8f5a6b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,271 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:03:40,271 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:03:40,272 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:03:40,274 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51802, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:03:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b383f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@572ebf91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,285 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:03:40,285 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:03:40,294 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:03:40,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:03:40,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:03:40,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175526a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:03:40,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:03:40,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:03:40,303 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:03:40,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f55d533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46758a4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:03:40,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:03:40,320 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:03:40,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:03:40,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:03:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1d7645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:03:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:03:40,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:03:40,324 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:03:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bc6ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5768c1ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,333 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:03:40,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:03:40,342 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:03:40,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:03:40,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:03:40,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e153ddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:03:40,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:03:40,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:03:40,351 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:03:40,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35159008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,357 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cc494e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,357 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:03:40,357 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:03:40,358 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:03:40,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:03:40,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:03:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0fbd8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:03:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:03:40,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:03:40,362 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:03:40,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75abeb9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65526cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,366 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:03:40,367 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:03:40,368 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:03:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:03:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:03:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@598c3d89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:03:40,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:03:40,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:03:40,371 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:03:40,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59723d19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@522561b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,377 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:03:40,377 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:03:40,378 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:03:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:03:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:03:40,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e2fecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:03:40,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:03:40,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:03:40,380 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:03:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4501fb62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,385 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5864d7ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,385 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:03:40,385 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:03:40,386 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:03:40,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:03:40,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:03:40,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b3b865c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:03:40,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:03:40,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:03:40,389 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:03:40,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13210d53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2299c977, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,391 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:03:40,391 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:03:40,392 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:03:40,392 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:03:40,393 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:03:40,393 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6602a8c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,393 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:03:40,393 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:03:40,393 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:03:40,394 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:03:40,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e7933c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:03:40,402 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:03:40,404 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:03:40,404 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:03:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:03:40,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:03:40,408 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:03:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:03:40,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:03:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-25T12:03:40,412 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:03:40,413 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:03:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T12:03:40,415 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:03:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:03:40,417 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:03:40,418 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:03:40,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:03:40,420 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:03:40,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:03:40,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:03:40,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:03:40,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:03:40,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:03:40,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:03:40,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:03:40,427 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:03:40,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:03:40,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:03:40,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:03:40,434 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:03:40,438 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:03:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:03:40,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:03:40,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:03:40,446 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:03:40,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:03:40,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:03:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:03:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:03:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:03:40,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:03:40,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:03:40,451 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:03:40,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:03:40,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:03:40,455 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:03:40,457 DEBUG [Thread-158 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:03:40,458 DEBUG [Thread-158 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:03:40,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:03:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:03:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:03:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T12:03:40,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-25T12:03:40,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:40,622 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-25T12:03:40,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T12:03:40,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/be598eaa5304408aa2ea53463fb8dfcf is 50, key is test_row_0/A:col10/1732536220567/Put/seqid=0 2024-11-25T12:03:40,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741837_1013 (size=12001) 2024-11-25T12:03:40,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:40,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536280865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:40,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536280862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:40,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:40,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:40,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:40,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:40,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:40,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536280873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:40,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:40,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536280894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:40,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:40,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:40,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:40,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:40,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:40,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:40,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:40,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536280893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:40,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:41,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536281017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:41,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536281017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:41,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536281028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:41,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536281030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:41,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536281030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:41,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T12:03:41,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:41,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536281237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:41,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:41,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:41,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536281237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:41,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:41,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536281248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:41,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536281248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:41,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:41,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:41,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536281249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:41,292 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/be598eaa5304408aa2ea53463fb8dfcf 2024-11-25T12:03:41,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/68f75de0725e46bd8ddb6b44292db658 is 50, key is test_row_0/B:col10/1732536220567/Put/seqid=0 2024-11-25T12:03:41,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741838_1014 (size=12001) 2024-11-25T12:03:41,548 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/68f75de0725e46bd8ddb6b44292db658 2024-11-25T12:03:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T12:03:41,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536281549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:41,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:41,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:41,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:41,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536281567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:41,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536281569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536281557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:41,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:41,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:41,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536281569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:41,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:41,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:41,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/e196cced833b4c5e93a16cfb31eeba7b is 50, key is test_row_0/C:col10/1732536220567/Put/seqid=0 2024-11-25T12:03:41,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741839_1015 (size=12001) 2024-11-25T12:03:41,686 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/e196cced833b4c5e93a16cfb31eeba7b 2024-11-25T12:03:41,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/be598eaa5304408aa2ea53463fb8dfcf as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/be598eaa5304408aa2ea53463fb8dfcf 2024-11-25T12:03:41,743 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/be598eaa5304408aa2ea53463fb8dfcf, entries=150, sequenceid=11, filesize=11.7 K 2024-11-25T12:03:41,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/68f75de0725e46bd8ddb6b44292db658 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/68f75de0725e46bd8ddb6b44292db658 2024-11-25T12:03:41,777 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/68f75de0725e46bd8ddb6b44292db658, entries=150, sequenceid=11, filesize=11.7 K 2024-11-25T12:03:41,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/e196cced833b4c5e93a16cfb31eeba7b as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/e196cced833b4c5e93a16cfb31eeba7b 2024-11-25T12:03:41,815 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/e196cced833b4c5e93a16cfb31eeba7b, entries=150, sequenceid=11, filesize=11.7 K 2024-11-25T12:03:41,824 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 057d2058e8441af178fff3bc63c12b69 in 1204ms, sequenceid=11, compaction requested=false 2024-11-25T12:03:41,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-25T12:03:41,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:41,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:41,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-25T12:03:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-25T12:03:41,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T12:03:41,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4380 sec 2024-11-25T12:03:41,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.4610 sec 2024-11-25T12:03:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:42,091 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:03:42,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-25T12:03:42,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/8113cc61c64945baa16ec7baa26d62e3 is 50, key is test_row_0/A:col10/1732536220885/Put/seqid=0 2024-11-25T12:03:42,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:42,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536282106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:42,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536282118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:42,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:42,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741840_1016 (size=12001) 2024-11-25T12:03:42,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:42,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536282149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536282155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:42,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:42,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536282155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:42,185 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:42,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:42,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536282257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:42,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:42,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536282256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:42,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:42,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536282287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:42,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:42,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:42,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536282298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:42,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:42,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:42,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536282298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:42,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536282477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:42,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536282478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:42,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:42,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:42,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536282509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:42,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:42,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536282517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:42,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:42,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536282517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:42,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T12:03:42,577 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:03:42,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:03:42,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/8113cc61c64945baa16ec7baa26d62e3 2024-11-25T12:03:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-25T12:03:42,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T12:03:42,587 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:03:42,589 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:03:42,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:03:42,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/c17a8d757e6244dab1615cd644265c42 is 50, key is test_row_0/B:col10/1732536220885/Put/seqid=0 2024-11-25T12:03:42,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741841_1017 (size=12001) 2024-11-25T12:03:42,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/c17a8d757e6244dab1615cd644265c42 2024-11-25T12:03:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T12:03:42,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/c84bfb37fca44df595a3323345869c80 is 50, key is test_row_0/C:col10/1732536220885/Put/seqid=0 2024-11-25T12:03:42,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-25T12:03:42,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:42,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:42,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:42,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:42,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:42,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:42,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536282787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:42,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:42,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536282799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:42,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:42,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741842_1018 (size=12001) 2024-11-25T12:03:42,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536282827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:42,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:42,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536282837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:42,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:42,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536282840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:42,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:42,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T12:03:42,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-25T12:03:42,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:42,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:42,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:42,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:42,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:43,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-25T12:03:43,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:43,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:43,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:43,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:43,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:43,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T12:03:43,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/c84bfb37fca44df595a3323345869c80 2024-11-25T12:03:43,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/8113cc61c64945baa16ec7baa26d62e3 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/8113cc61c64945baa16ec7baa26d62e3 2024-11-25T12:03:43,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/8113cc61c64945baa16ec7baa26d62e3, entries=150, sequenceid=39, filesize=11.7 K 2024-11-25T12:03:43,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/c17a8d757e6244dab1615cd644265c42 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/c17a8d757e6244dab1615cd644265c42 2024-11-25T12:03:43,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/c17a8d757e6244dab1615cd644265c42, entries=150, sequenceid=39, filesize=11.7 K 2024-11-25T12:03:43,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/c84bfb37fca44df595a3323345869c80 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/c84bfb37fca44df595a3323345869c80 2024-11-25T12:03:43,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-25T12:03:43,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/c84bfb37fca44df595a3323345869c80, entries=150, sequenceid=39, filesize=11.7 K 2024-11-25T12:03:43,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 057d2058e8441af178fff3bc63c12b69 in 1224ms, sequenceid=39, compaction requested=false 2024-11-25T12:03:43,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:43,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:43,317 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-25T12:03:43,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/e4f58bc6c0b747a78ed6c454dd7edc36 is 50, key is test_row_1/A:col10/1732536222132/Put/seqid=0 2024-11-25T12:03:43,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:43,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:43,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741843_1019 (size=9657) 2024-11-25T12:03:43,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:43,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536283524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:43,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:43,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:43,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:43,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536283541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:43,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536283550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:43,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:43,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536283563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:43,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:43,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536283570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:43,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536283678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:43,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:43,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536283707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:43,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:43,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:43,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536283718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536283718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T12:03:43,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536283728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:43,780 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/e4f58bc6c0b747a78ed6c454dd7edc36 2024-11-25T12:03:43,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/518ea8c508c145a2989449e309093be4 is 50, key is test_row_1/B:col10/1732536222132/Put/seqid=0 2024-11-25T12:03:43,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741844_1020 (size=9657) 2024-11-25T12:03:43,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536283927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:43,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:43,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536283937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:43,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:43,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536283937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:43,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:43,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536283937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:43,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536283956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:43,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:44,090 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T12:03:44,234 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T12:03:44,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536284246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:44,255 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T12:03:44,256 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-25T12:03:44,258 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T12:03:44,258 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-25T12:03:44,259 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-25T12:03:44,259 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-25T12:03:44,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T12:03:44,260 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-25T12:03:44,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536284257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:44,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536284257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:44,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536284258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:44,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536284284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:44,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:44,296 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/518ea8c508c145a2989449e309093be4 2024-11-25T12:03:44,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/63502da6c822477a80e776f337720cd8 is 50, key is test_row_1/C:col10/1732536222132/Put/seqid=0 2024-11-25T12:03:44,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741845_1021 (size=9657) 2024-11-25T12:03:44,367 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/63502da6c822477a80e776f337720cd8 2024-11-25T12:03:44,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/e4f58bc6c0b747a78ed6c454dd7edc36 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/e4f58bc6c0b747a78ed6c454dd7edc36 2024-11-25T12:03:44,422 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/e4f58bc6c0b747a78ed6c454dd7edc36, entries=100, sequenceid=47, filesize=9.4 K 2024-11-25T12:03:44,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/518ea8c508c145a2989449e309093be4 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/518ea8c508c145a2989449e309093be4 2024-11-25T12:03:44,441 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/518ea8c508c145a2989449e309093be4, entries=100, sequenceid=47, filesize=9.4 K 2024-11-25T12:03:44,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/63502da6c822477a80e776f337720cd8 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/63502da6c822477a80e776f337720cd8 2024-11-25T12:03:44,469 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/63502da6c822477a80e776f337720cd8, entries=100, sequenceid=47, filesize=9.4 K 2024-11-25T12:03:44,481 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 057d2058e8441af178fff3bc63c12b69 in 1164ms, sequenceid=47, compaction requested=true 2024-11-25T12:03:44,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:44,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:44,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-25T12:03:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-25T12:03:44,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-25T12:03:44,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8910 sec 2024-11-25T12:03:44,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.9130 sec 2024-11-25T12:03:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-25T12:03:44,758 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:03:44,759 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:03:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-25T12:03:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T12:03:44,768 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:03:44,773 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:03:44,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:03:44,774 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:03:44,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-25T12:03:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:44,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/244ba09f29f242678c5bca4f96f072d7 is 50, key is test_row_0/A:col10/1732536223534/Put/seqid=0 2024-11-25T12:03:44,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:44,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:44,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536284791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:44,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:44,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536284788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:44,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:44,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:44,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536284796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:44,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:44,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:44,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536284798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:44,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:44,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536284801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:44,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741846_1022 (size=14341) 2024-11-25T12:03:44,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/244ba09f29f242678c5bca4f96f072d7 2024-11-25T12:03:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T12:03:44,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/2dffcd7f8b444eb08434c21d1a4755ee is 50, key is test_row_0/B:col10/1732536223534/Put/seqid=0 2024-11-25T12:03:44,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741847_1023 (size=12001) 2024-11-25T12:03:44,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:44,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536284906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:44,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:44,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536284917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:44,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:44,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536284917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:44,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:44,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536284918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:44,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T12:03:44,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:44,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:44,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:44,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:44,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:44,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:44,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:44,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T12:03:45,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T12:03:45,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:45,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:45,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:45,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:45,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:45,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536285131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:45,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:45,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:45,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:45,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536285148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:45,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:45,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536285148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:45,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:45,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:45,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:45,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536285148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:45,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:45,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:45,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:45,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:45,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:45,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:45,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T12:03:45,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:45,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:45,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:45,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:45,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:45,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:45,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/2dffcd7f8b444eb08434c21d1a4755ee 2024-11-25T12:03:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T12:03:45,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/2141ed26e5d14f118e095020fa141fc6 is 50, key is test_row_0/C:col10/1732536223534/Put/seqid=0 2024-11-25T12:03:45,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T12:03:45,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:45,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:45,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:45,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:45,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:45,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:45,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536285449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:45,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:45,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:45,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741848_1024 (size=12001) 2024-11-25T12:03:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536285468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536285467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:45,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:45,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:45,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:45,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:45,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:45,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:45,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536285482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:45,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:45,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:45,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:45,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T12:03:45,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:45,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:45,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:45,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:45,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:45,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T12:03:45,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:45,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:45,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:45,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:45,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:45,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:45,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536285817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:45,829 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_0' in 057d2058e8441af178fff3bc63c12b69 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:03:45,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:45,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:45,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:45,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/2141ed26e5d14f118e095020fa141fc6 2024-11-25T12:03:45,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/244ba09f29f242678c5bca4f96f072d7 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/244ba09f29f242678c5bca4f96f072d7 2024-11-25T12:03:45,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/244ba09f29f242678c5bca4f96f072d7, entries=200, sequenceid=76, filesize=14.0 K 2024-11-25T12:03:45,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/2dffcd7f8b444eb08434c21d1a4755ee as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/2dffcd7f8b444eb08434c21d1a4755ee 2024-11-25T12:03:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T12:03:45,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/2dffcd7f8b444eb08434c21d1a4755ee, entries=150, sequenceid=76, filesize=11.7 K 2024-11-25T12:03:45,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/2141ed26e5d14f118e095020fa141fc6 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/2141ed26e5d14f118e095020fa141fc6 2024-11-25T12:03:45,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T12:03:45,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:45,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:45,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:45,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:45,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:45,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/2141ed26e5d14f118e095020fa141fc6, entries=150, sequenceid=76, filesize=11.7 K 2024-11-25T12:03:45,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 057d2058e8441af178fff3bc63c12b69 in 1176ms, sequenceid=76, compaction requested=true 2024-11-25T12:03:45,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:45,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:03:45,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:45,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:03:45,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:45,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:03:45,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:03:45,962 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:03:45,967 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:03:45,969 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/B is initiating minor compaction (all files) 2024-11-25T12:03:45,969 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/B in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:45,970 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/68f75de0725e46bd8ddb6b44292db658, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/c17a8d757e6244dab1615cd644265c42, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/518ea8c508c145a2989449e309093be4, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/2dffcd7f8b444eb08434c21d1a4755ee] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=44.6 K 2024-11-25T12:03:45,971 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 68f75de0725e46bd8ddb6b44292db658, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732536220475 2024-11-25T12:03:45,972 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting c17a8d757e6244dab1615cd644265c42, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732536220864 2024-11-25T12:03:45,973 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 518ea8c508c145a2989449e309093be4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732536222094 2024-11-25T12:03:45,974 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dffcd7f8b444eb08434c21d1a4755ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732536223534 2024-11-25T12:03:45,975 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:03:45,983 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48000 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:03:45,983 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/A is initiating minor compaction (all files) 2024-11-25T12:03:45,983 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/A in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:45,983 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/be598eaa5304408aa2ea53463fb8dfcf, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/8113cc61c64945baa16ec7baa26d62e3, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/e4f58bc6c0b747a78ed6c454dd7edc36, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/244ba09f29f242678c5bca4f96f072d7] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=46.9 K 2024-11-25T12:03:45,993 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting be598eaa5304408aa2ea53463fb8dfcf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732536220475 2024-11-25T12:03:45,994 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8113cc61c64945baa16ec7baa26d62e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732536220864 2024-11-25T12:03:45,994 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4f58bc6c0b747a78ed6c454dd7edc36, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732536222094 2024-11-25T12:03:45,995 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 244ba09f29f242678c5bca4f96f072d7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732536223525 2024-11-25T12:03:46,028 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#B#compaction#12 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:46,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:46,032 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:03:46,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:03:46,038 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/74fd1c6ebc8f4d53b2db05a05b6018a2 is 50, key is test_row_0/B:col10/1732536223534/Put/seqid=0 2024-11-25T12:03:46,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741849_1025 (size=12139) 2024-11-25T12:03:46,071 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#A#compaction#13 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:46,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/3b6f93352e8f43aab4edb06c6cd2e5ac is 50, key is test_row_0/A:col10/1732536226032/Put/seqid=0 2024-11-25T12:03:46,081 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/c7215971628d4c3fb112c59dbcdb0133 is 50, key is test_row_0/A:col10/1732536223534/Put/seqid=0 2024-11-25T12:03:46,086 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/74fd1c6ebc8f4d53b2db05a05b6018a2 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/74fd1c6ebc8f4d53b2db05a05b6018a2 2024-11-25T12:03:46,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T12:03:46,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:46,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:46,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:46,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:46,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:46,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:46,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741850_1026 (size=12001) 2024-11-25T12:03:46,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741851_1027 (size=12139) 2024-11-25T12:03:46,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/3b6f93352e8f43aab4edb06c6cd2e5ac 2024-11-25T12:03:46,136 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/B of 057d2058e8441af178fff3bc63c12b69 into 74fd1c6ebc8f4d53b2db05a05b6018a2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:46,136 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:46,142 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/B, priority=12, startTime=1732536225955; duration=0sec 2024-11-25T12:03:46,143 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:03:46,143 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:B 2024-11-25T12:03:46,143 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:03:46,150 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/c7215971628d4c3fb112c59dbcdb0133 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/c7215971628d4c3fb112c59dbcdb0133 2024-11-25T12:03:46,156 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:03:46,157 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/C is initiating minor compaction (all files) 2024-11-25T12:03:46,157 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/C in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:46,157 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/e196cced833b4c5e93a16cfb31eeba7b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/c84bfb37fca44df595a3323345869c80, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/63502da6c822477a80e776f337720cd8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/2141ed26e5d14f118e095020fa141fc6] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=44.6 K 2024-11-25T12:03:46,158 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting e196cced833b4c5e93a16cfb31eeba7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732536220475 2024-11-25T12:03:46,159 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting c84bfb37fca44df595a3323345869c80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732536220864 2024-11-25T12:03:46,160 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 63502da6c822477a80e776f337720cd8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732536222094 2024-11-25T12:03:46,161 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 2141ed26e5d14f118e095020fa141fc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732536223534 2024-11-25T12:03:46,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/7e5232390e3245269f5f4d2ce59aa807 is 50, key is test_row_0/B:col10/1732536226032/Put/seqid=0 2024-11-25T12:03:46,175 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/A of 057d2058e8441af178fff3bc63c12b69 into c7215971628d4c3fb112c59dbcdb0133(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:46,175 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:46,175 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/A, priority=12, startTime=1732536225952; duration=0sec 2024-11-25T12:03:46,176 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:46,176 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:A 2024-11-25T12:03:46,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:46,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536286168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:46,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:46,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536286169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:46,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:46,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536286172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:46,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:46,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:46,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536286186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:46,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741852_1028 (size=12001) 2024-11-25T12:03:46,206 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#C#compaction#16 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:46,207 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/2c8bcf09bf8b4c7c8a555ab747a72db3 is 50, key is test_row_0/C:col10/1732536223534/Put/seqid=0 2024-11-25T12:03:46,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/7e5232390e3245269f5f4d2ce59aa807 2024-11-25T12:03:46,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T12:03:46,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:46,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:46,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:46,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:46,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/55a08b11b17c4113a3fa3d777a0296f9 is 50, key is test_row_0/C:col10/1732536226032/Put/seqid=0 2024-11-25T12:03:46,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:46,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741853_1029 (size=12139) 2024-11-25T12:03:46,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741854_1030 (size=12001) 2024-11-25T12:03:46,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/55a08b11b17c4113a3fa3d777a0296f9 2024-11-25T12:03:46,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536286298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:46,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:46,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536286298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536286298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:46,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536286306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:46,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/3b6f93352e8f43aab4edb06c6cd2e5ac as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/3b6f93352e8f43aab4edb06c6cd2e5ac 2024-11-25T12:03:46,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/3b6f93352e8f43aab4edb06c6cd2e5ac, entries=150, sequenceid=89, filesize=11.7 K 2024-11-25T12:03:46,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/7e5232390e3245269f5f4d2ce59aa807 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7e5232390e3245269f5f4d2ce59aa807 2024-11-25T12:03:46,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7e5232390e3245269f5f4d2ce59aa807, entries=150, sequenceid=89, filesize=11.7 K 2024-11-25T12:03:46,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/55a08b11b17c4113a3fa3d777a0296f9 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/55a08b11b17c4113a3fa3d777a0296f9 2024-11-25T12:03:46,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/55a08b11b17c4113a3fa3d777a0296f9, entries=150, sequenceid=89, filesize=11.7 K 2024-11-25T12:03:46,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 057d2058e8441af178fff3bc63c12b69 in 339ms, sequenceid=89, compaction requested=false 2024-11-25T12:03:46,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:46,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-25T12:03:46,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:46,409 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-25T12:03:46,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/d5656e5dfd14433898e04907b74420f2 is 50, key is test_row_0/A:col10/1732536226165/Put/seqid=0 2024-11-25T12:03:46,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741855_1031 (size=12001) 2024-11-25T12:03:46,440 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/d5656e5dfd14433898e04907b74420f2 2024-11-25T12:03:46,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/bfa1699ec4c745599838c8adb0d111e1 is 50, key is test_row_0/B:col10/1732536226165/Put/seqid=0 2024-11-25T12:03:46,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741856_1032 (size=12001) 2024-11-25T12:03:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:46,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:46,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536286569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:46,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:46,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536286577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:46,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:46,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536286589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:46,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536286604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:46,677 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/2c8bcf09bf8b4c7c8a555ab747a72db3 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/2c8bcf09bf8b4c7c8a555ab747a72db3 2024-11-25T12:03:46,707 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/C of 057d2058e8441af178fff3bc63c12b69 into 2c8bcf09bf8b4c7c8a555ab747a72db3(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:46,707 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:46,707 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/C, priority=12, startTime=1732536225956; duration=0sec 2024-11-25T12:03:46,709 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:46,709 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:C 2024-11-25T12:03:46,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536286700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:46,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536286697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:46,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536286726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:46,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:46,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536286726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:46,913 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/bfa1699ec4c745599838c8adb0d111e1 2024-11-25T12:03:46,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:46,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536286929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:46,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T12:03:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:46,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:46,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536286937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:46,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:46,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:46,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536286947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:46,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:46,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:46,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536286959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:46,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:46,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:46,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/16da9fbbf0aa4ee3a1596654a2aa2c7d is 50, key is test_row_0/C:col10/1732536226165/Put/seqid=0 2024-11-25T12:03:47,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741857_1033 (size=12001) 2024-11-25T12:03:47,026 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/16da9fbbf0aa4ee3a1596654a2aa2c7d 2024-11-25T12:03:47,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/d5656e5dfd14433898e04907b74420f2 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/d5656e5dfd14433898e04907b74420f2 2024-11-25T12:03:47,057 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/d5656e5dfd14433898e04907b74420f2, entries=150, sequenceid=114, filesize=11.7 K 2024-11-25T12:03:47,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/bfa1699ec4c745599838c8adb0d111e1 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/bfa1699ec4c745599838c8adb0d111e1 2024-11-25T12:03:47,069 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/bfa1699ec4c745599838c8adb0d111e1, entries=150, sequenceid=114, filesize=11.7 K 2024-11-25T12:03:47,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/16da9fbbf0aa4ee3a1596654a2aa2c7d as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/16da9fbbf0aa4ee3a1596654a2aa2c7d 2024-11-25T12:03:47,085 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/16da9fbbf0aa4ee3a1596654a2aa2c7d, entries=150, sequenceid=114, filesize=11.7 K 2024-11-25T12:03:47,093 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 057d2058e8441af178fff3bc63c12b69 in 684ms, sequenceid=114, compaction requested=true 2024-11-25T12:03:47,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:47,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:47,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-25T12:03:47,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-25T12:03:47,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-25T12:03:47,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3230 sec 2024-11-25T12:03:47,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.3430 sec 2024-11-25T12:03:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:47,259 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:03:47,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-25T12:03:47,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/e0be88cbf187412eb2bd7b4e1c7d862e is 50, key is test_row_0/A:col10/1732536227251/Put/seqid=0 2024-11-25T12:03:47,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741858_1034 (size=12101) 2024-11-25T12:03:47,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/e0be88cbf187412eb2bd7b4e1c7d862e 2024-11-25T12:03:47,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/bb74317b3d31480b817b048daff7ba41 is 50, key is test_row_0/B:col10/1732536227251/Put/seqid=0 2024-11-25T12:03:47,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536287351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:47,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:47,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536287356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:47,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:47,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:47,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536287361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536287360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:47,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:47,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741859_1035 (size=12101) 2024-11-25T12:03:47,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536287467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:47,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536287492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:47,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536287492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:47,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:47,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536287490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:47,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:47,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536287689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:47,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:47,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536287707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:47,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536287708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:47,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536287708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:47,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/bb74317b3d31480b817b048daff7ba41 2024-11-25T12:03:47,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/5389a83e64484b41afe9f0c3520c16fc is 50, key is test_row_0/C:col10/1732536227251/Put/seqid=0 2024-11-25T12:03:47,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:47,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536287856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:47,865 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_0' in 057d2058e8441af178fff3bc63c12b69 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4295 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:03:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:47,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:47,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741860_1036 (size=12101) 2024-11-25T12:03:47,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/5389a83e64484b41afe9f0c3520c16fc 2024-11-25T12:03:47,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/e0be88cbf187412eb2bd7b4e1c7d862e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/e0be88cbf187412eb2bd7b4e1c7d862e 2024-11-25T12:03:47,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/e0be88cbf187412eb2bd7b4e1c7d862e, entries=150, sequenceid=132, filesize=11.8 K 2024-11-25T12:03:47,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/bb74317b3d31480b817b048daff7ba41 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/bb74317b3d31480b817b048daff7ba41 2024-11-25T12:03:48,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:48,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536287996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:48,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:48,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536288017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:48,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:48,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536288017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:48,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:48,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536288017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:48,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:48,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/bb74317b3d31480b817b048daff7ba41, entries=150, sequenceid=132, filesize=11.8 K 2024-11-25T12:03:48,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/5389a83e64484b41afe9f0c3520c16fc as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/5389a83e64484b41afe9f0c3520c16fc 2024-11-25T12:03:48,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/5389a83e64484b41afe9f0c3520c16fc, entries=150, sequenceid=132, filesize=11.8 K 2024-11-25T12:03:48,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 057d2058e8441af178fff3bc63c12b69 in 810ms, sequenceid=132, compaction requested=true 2024-11-25T12:03:48,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:48,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:03:48,069 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:03:48,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:48,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:03:48,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:03:48,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:03:48,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:03:48,070 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:03:48,073 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:03:48,073 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/A is initiating minor compaction (all files) 2024-11-25T12:03:48,073 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/A in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:48,073 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/c7215971628d4c3fb112c59dbcdb0133, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/3b6f93352e8f43aab4edb06c6cd2e5ac, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/d5656e5dfd14433898e04907b74420f2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/e0be88cbf187412eb2bd7b4e1c7d862e] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=47.1 K 2024-11-25T12:03:48,074 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:03:48,075 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/B is initiating minor compaction (all files) 2024-11-25T12:03:48,075 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/B in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:48,075 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/74fd1c6ebc8f4d53b2db05a05b6018a2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7e5232390e3245269f5f4d2ce59aa807, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/bfa1699ec4c745599838c8adb0d111e1, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/bb74317b3d31480b817b048daff7ba41] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=47.1 K 2024-11-25T12:03:48,076 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7215971628d4c3fb112c59dbcdb0133, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732536223534 2024-11-25T12:03:48,076 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 74fd1c6ebc8f4d53b2db05a05b6018a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732536223534 2024-11-25T12:03:48,078 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b6f93352e8f43aab4edb06c6cd2e5ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732536224792 2024-11-25T12:03:48,078 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e5232390e3245269f5f4d2ce59aa807, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732536224792 2024-11-25T12:03:48,078 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5656e5dfd14433898e04907b74420f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732536226138 2024-11-25T12:03:48,079 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting bfa1699ec4c745599838c8adb0d111e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732536226138 2024-11-25T12:03:48,079 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0be88cbf187412eb2bd7b4e1c7d862e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536226558 2024-11-25T12:03:48,080 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting bb74317b3d31480b817b048daff7ba41, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536226558 2024-11-25T12:03:48,127 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#A#compaction#24 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:48,128 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/a13bc658c7f34c0d809a54f6de511254 is 50, key is test_row_0/A:col10/1732536227251/Put/seqid=0 2024-11-25T12:03:48,139 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#B#compaction#25 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:48,140 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/e9d6c6e15c304d86a14ccb3cabd3e0a5 is 50, key is test_row_0/B:col10/1732536227251/Put/seqid=0 2024-11-25T12:03:48,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741862_1038 (size=12375) 2024-11-25T12:03:48,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741861_1037 (size=12375) 2024-11-25T12:03:48,226 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/e9d6c6e15c304d86a14ccb3cabd3e0a5 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/e9d6c6e15c304d86a14ccb3cabd3e0a5 2024-11-25T12:03:48,242 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/B of 057d2058e8441af178fff3bc63c12b69 into e9d6c6e15c304d86a14ccb3cabd3e0a5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:48,242 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:48,242 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/B, priority=12, startTime=1732536228069; duration=0sec 2024-11-25T12:03:48,242 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:03:48,242 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:B 2024-11-25T12:03:48,242 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:03:48,252 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:03:48,253 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/C is initiating minor compaction (all files) 2024-11-25T12:03:48,253 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/C in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:48,253 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/2c8bcf09bf8b4c7c8a555ab747a72db3, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/55a08b11b17c4113a3fa3d777a0296f9, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/16da9fbbf0aa4ee3a1596654a2aa2c7d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/5389a83e64484b41afe9f0c3520c16fc] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=47.1 K 2024-11-25T12:03:48,254 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c8bcf09bf8b4c7c8a555ab747a72db3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732536223534 2024-11-25T12:03:48,254 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 55a08b11b17c4113a3fa3d777a0296f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732536224792 2024-11-25T12:03:48,255 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 16da9fbbf0aa4ee3a1596654a2aa2c7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732536226138 2024-11-25T12:03:48,256 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 5389a83e64484b41afe9f0c3520c16fc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536226558 2024-11-25T12:03:48,288 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#C#compaction#26 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:48,289 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/575bf187480d4ee5861512eca7664197 is 50, key is test_row_0/C:col10/1732536227251/Put/seqid=0 2024-11-25T12:03:48,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741863_1039 (size=12375) 2024-11-25T12:03:48,368 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/575bf187480d4ee5861512eca7664197 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/575bf187480d4ee5861512eca7664197 2024-11-25T12:03:48,381 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/C of 057d2058e8441af178fff3bc63c12b69 into 575bf187480d4ee5861512eca7664197(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:48,382 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:48,382 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/C, priority=12, startTime=1732536228070; duration=0sec 2024-11-25T12:03:48,382 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:48,382 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:C 2024-11-25T12:03:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:48,520 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:03:48,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-25T12:03:48,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/50b2e40a15cb471fb9e619f81e434a24 is 50, key is test_row_0/A:col10/1732536227356/Put/seqid=0 2024-11-25T12:03:48,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741864_1040 (size=12151) 2024-11-25T12:03:48,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/50b2e40a15cb471fb9e619f81e434a24 2024-11-25T12:03:48,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:48,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536288574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536288576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:48,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:48,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536288590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:48,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536288590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:48,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:48,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/881068bb4d4a47a2af7ac45ae72f0c54 is 50, key is test_row_0/B:col10/1732536227356/Put/seqid=0 2024-11-25T12:03:48,629 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/a13bc658c7f34c0d809a54f6de511254 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/a13bc658c7f34c0d809a54f6de511254 2024-11-25T12:03:48,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741865_1041 (size=12151) 2024-11-25T12:03:48,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/881068bb4d4a47a2af7ac45ae72f0c54 2024-11-25T12:03:48,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/5f1230b505c2475fa079c5b9b986292f is 50, key is test_row_0/C:col10/1732536227356/Put/seqid=0 2024-11-25T12:03:48,670 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/A of 057d2058e8441af178fff3bc63c12b69 into a13bc658c7f34c0d809a54f6de511254(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:48,671 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:48,671 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/A, priority=12, startTime=1732536228069; duration=0sec 2024-11-25T12:03:48,671 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:48,671 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:A 2024-11-25T12:03:48,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:48,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536288696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:48,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536288697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:48,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:48,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741866_1042 (size=12151) 2024-11-25T12:03:48,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/5f1230b505c2475fa079c5b9b986292f 2024-11-25T12:03:48,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:48,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536288717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:48,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:48,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536288720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:48,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/50b2e40a15cb471fb9e619f81e434a24 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/50b2e40a15cb471fb9e619f81e434a24 2024-11-25T12:03:48,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/50b2e40a15cb471fb9e619f81e434a24, entries=150, sequenceid=154, filesize=11.9 K 2024-11-25T12:03:48,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/881068bb4d4a47a2af7ac45ae72f0c54 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/881068bb4d4a47a2af7ac45ae72f0c54 2024-11-25T12:03:48,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/881068bb4d4a47a2af7ac45ae72f0c54, entries=150, sequenceid=154, filesize=11.9 K 2024-11-25T12:03:48,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/5f1230b505c2475fa079c5b9b986292f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/5f1230b505c2475fa079c5b9b986292f 2024-11-25T12:03:48,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/5f1230b505c2475fa079c5b9b986292f, entries=150, sequenceid=154, filesize=11.9 K 2024-11-25T12:03:48,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 057d2058e8441af178fff3bc63c12b69 in 291ms, sequenceid=154, compaction requested=false 2024-11-25T12:03:48,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:48,911 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:03:48,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-25T12:03:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:48,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/218536f9204b46d49460a35b01666e9d is 50, key is test_row_0/A:col10/1732536228584/Put/seqid=0 2024-11-25T12:03:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-25T12:03:48,949 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:03:48,951 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:03:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-25T12:03:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T12:03:48,955 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:03:48,960 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:03:48,961 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:03:48,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741867_1043 (size=14541) 2024-11-25T12:03:48,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:48,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536288982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:48,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536288984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:48,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:48,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536288986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:48,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:48,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:48,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536288995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T12:03:49,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:49,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536289106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:49,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:49,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536289107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:49,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:49,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536289107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:49,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,116 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:49,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536289112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:49,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:49,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T12:03:49,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:49,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:49,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:49,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:49,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:49,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:49,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T12:03:49,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:49,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:49,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:49,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:49,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:49,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T12:03:49,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:49,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:49,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536289317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:49,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:49,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:49,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536289326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:49,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536289327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:49,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:49,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536289329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:49,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/218536f9204b46d49460a35b01666e9d 2024-11-25T12:03:49,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/f52905948f4e44a9b3db268b69ad0174 is 50, key is test_row_0/B:col10/1732536228584/Put/seqid=0 2024-11-25T12:03:49,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T12:03:49,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:49,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:49,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:49,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:49,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:49,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741868_1044 (size=12151) 2024-11-25T12:03:49,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/f52905948f4e44a9b3db268b69ad0174 2024-11-25T12:03:49,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/8a4fe0509c554d49af145ac7d333b908 is 50, key is test_row_0/C:col10/1732536228584/Put/seqid=0 2024-11-25T12:03:49,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741869_1045 (size=12151) 2024-11-25T12:03:49,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/8a4fe0509c554d49af145ac7d333b908 2024-11-25T12:03:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T12:03:49,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T12:03:49,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:49,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:49,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:49,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:49,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:49,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/218536f9204b46d49460a35b01666e9d as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/218536f9204b46d49460a35b01666e9d 2024-11-25T12:03:49,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536289626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:49,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:49,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536289637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:49,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:49,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:49,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536289637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:49,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:49,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536289647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:49,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:49,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/218536f9204b46d49460a35b01666e9d, entries=200, sequenceid=174, filesize=14.2 K 2024-11-25T12:03:49,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/f52905948f4e44a9b3db268b69ad0174 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/f52905948f4e44a9b3db268b69ad0174 2024-11-25T12:03:49,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/f52905948f4e44a9b3db268b69ad0174, entries=150, sequenceid=174, filesize=11.9 K 2024-11-25T12:03:49,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/8a4fe0509c554d49af145ac7d333b908 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/8a4fe0509c554d49af145ac7d333b908 2024-11-25T12:03:49,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/8a4fe0509c554d49af145ac7d333b908, entries=150, sequenceid=174, filesize=11.9 K 2024-11-25T12:03:49,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 057d2058e8441af178fff3bc63c12b69 in 817ms, sequenceid=174, compaction requested=true 2024-11-25T12:03:49,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:49,729 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:03:49,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:03:49,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:49,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:03:49,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:49,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:03:49,730 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:03:49,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:03:49,731 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39067 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:03:49,732 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/A is initiating minor compaction (all files) 2024-11-25T12:03:49,732 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/A in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:49,732 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/a13bc658c7f34c0d809a54f6de511254, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/50b2e40a15cb471fb9e619f81e434a24, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/218536f9204b46d49460a35b01666e9d] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=38.2 K 2024-11-25T12:03:49,737 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:03:49,737 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting a13bc658c7f34c0d809a54f6de511254, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536226558 2024-11-25T12:03:49,737 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/B is initiating minor compaction (all files) 2024-11-25T12:03:49,737 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/B in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:49,737 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/e9d6c6e15c304d86a14ccb3cabd3e0a5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/881068bb4d4a47a2af7ac45ae72f0c54, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/f52905948f4e44a9b3db268b69ad0174] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=35.8 K 2024-11-25T12:03:49,741 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting e9d6c6e15c304d86a14ccb3cabd3e0a5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536226558 2024-11-25T12:03:49,741 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50b2e40a15cb471fb9e619f81e434a24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732536227290 2024-11-25T12:03:49,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-25T12:03:49,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:49,745 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 881068bb4d4a47a2af7ac45ae72f0c54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732536227290 2024-11-25T12:03:49,745 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-25T12:03:49,745 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 218536f9204b46d49460a35b01666e9d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732536228584 2024-11-25T12:03:49,750 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting f52905948f4e44a9b3db268b69ad0174, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732536228584 2024-11-25T12:03:49,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/b561ff7d029a4012b60ffade52ca4bf0 is 50, key is test_row_0/A:col10/1732536228981/Put/seqid=0 2024-11-25T12:03:49,783 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#A#compaction#34 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:49,783 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#B#compaction#35 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:49,783 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/caa9234026f24447a7b4b95afa20c9f9 is 50, key is test_row_0/A:col10/1732536228584/Put/seqid=0 2024-11-25T12:03:49,783 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/bb0f252910ab42bcac0f275328c166f8 is 50, key is test_row_0/B:col10/1732536228584/Put/seqid=0 2024-11-25T12:03:49,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741870_1046 (size=12151) 2024-11-25T12:03:49,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741871_1047 (size=12527) 2024-11-25T12:03:49,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741872_1048 (size=12527) 2024-11-25T12:03:49,882 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/bb0f252910ab42bcac0f275328c166f8 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/bb0f252910ab42bcac0f275328c166f8 2024-11-25T12:03:49,895 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/caa9234026f24447a7b4b95afa20c9f9 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/caa9234026f24447a7b4b95afa20c9f9 2024-11-25T12:03:49,906 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/B of 057d2058e8441af178fff3bc63c12b69 into bb0f252910ab42bcac0f275328c166f8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:49,906 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:49,906 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/B, priority=13, startTime=1732536229730; duration=0sec 2024-11-25T12:03:49,907 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:03:49,907 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:B 2024-11-25T12:03:49,907 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:03:49,909 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:03:49,909 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/C is initiating minor compaction (all files) 2024-11-25T12:03:49,909 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/C in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:49,910 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/575bf187480d4ee5861512eca7664197, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/5f1230b505c2475fa079c5b9b986292f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/8a4fe0509c554d49af145ac7d333b908] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=35.8 K 2024-11-25T12:03:49,910 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 575bf187480d4ee5861512eca7664197, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536226558 2024-11-25T12:03:49,911 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/A of 057d2058e8441af178fff3bc63c12b69 into caa9234026f24447a7b4b95afa20c9f9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:49,911 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:49,911 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/A, priority=13, startTime=1732536229729; duration=0sec 2024-11-25T12:03:49,911 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f1230b505c2475fa079c5b9b986292f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732536227290 2024-11-25T12:03:49,912 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:49,912 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:A 2024-11-25T12:03:49,912 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a4fe0509c554d49af145ac7d333b908, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732536228584 2024-11-25T12:03:49,950 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#C#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:49,951 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/54d99e1e87af471f8cb1f08328f45967 is 50, key is test_row_0/C:col10/1732536228584/Put/seqid=0 2024-11-25T12:03:49,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741873_1049 (size=12527) 2024-11-25T12:03:50,000 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/54d99e1e87af471f8cb1f08328f45967 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/54d99e1e87af471f8cb1f08328f45967 2024-11-25T12:03:50,015 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/C of 057d2058e8441af178fff3bc63c12b69 into 54d99e1e87af471f8cb1f08328f45967(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:50,015 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:50,015 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/C, priority=13, startTime=1732536229730; duration=0sec 2024-11-25T12:03:50,015 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:50,016 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:C 2024-11-25T12:03:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T12:03:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:50,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:50,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:50,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536290198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:50,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:50,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:50,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536290199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:50,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536290200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:50,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:50,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536290209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:50,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:50,232 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/b561ff7d029a4012b60ffade52ca4bf0 2024-11-25T12:03:50,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/7bb1e4719f3b4fa9b9122ee81b97adc8 is 50, key is test_row_0/B:col10/1732536228981/Put/seqid=0 2024-11-25T12:03:50,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:50,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536290318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:50,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:50,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536290318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:50,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:50,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536290318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:50,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:50,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:50,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536290319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:50,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:50,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741874_1050 (size=12151) 2024-11-25T12:03:50,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:50,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536290526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:50,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536290527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:50,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536290527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:50,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:50,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536290527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:50,731 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/7bb1e4719f3b4fa9b9122ee81b97adc8 2024-11-25T12:03:50,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/dce8846486fa47d78d89499186fa189b is 50, key is test_row_0/C:col10/1732536228981/Put/seqid=0 2024-11-25T12:03:50,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741875_1051 (size=12151) 2024-11-25T12:03:50,820 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/dce8846486fa47d78d89499186fa189b 2024-11-25T12:03:50,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/b561ff7d029a4012b60ffade52ca4bf0 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/b561ff7d029a4012b60ffade52ca4bf0 2024-11-25T12:03:50,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:50,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:50,851 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/b561ff7d029a4012b60ffade52ca4bf0, entries=150, sequenceid=191, filesize=11.9 K 2024-11-25T12:03:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536290837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536290837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:50,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/7bb1e4719f3b4fa9b9122ee81b97adc8 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7bb1e4719f3b4fa9b9122ee81b97adc8 2024-11-25T12:03:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:50,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536290837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536290838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:50,867 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7bb1e4719f3b4fa9b9122ee81b97adc8, entries=150, sequenceid=191, filesize=11.9 K 2024-11-25T12:03:50,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/dce8846486fa47d78d89499186fa189b as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/dce8846486fa47d78d89499186fa189b 2024-11-25T12:03:50,896 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/dce8846486fa47d78d89499186fa189b, entries=150, sequenceid=191, filesize=11.9 K 2024-11-25T12:03:50,899 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 057d2058e8441af178fff3bc63c12b69 in 1154ms, sequenceid=191, compaction requested=false 2024-11-25T12:03:50,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:50,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:50,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-25T12:03:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-25T12:03:50,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-25T12:03:50,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9440 sec 2024-11-25T12:03:50,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.9590 sec 2024-11-25T12:03:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T12:03:51,106 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:03:51,107 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:03:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-25T12:03:51,111 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:03:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T12:03:51,113 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:03:51,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:03:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T12:03:51,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-25T12:03:51,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:51,268 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-25T12:03:51,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/5cb5a1b237fc4016a3e002cd97635661 is 50, key is test_row_0/A:col10/1732536230198/Put/seqid=0 2024-11-25T12:03:51,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741876_1052 (size=12151) 2024-11-25T12:03:51,294 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/5cb5a1b237fc4016a3e002cd97635661 2024-11-25T12:03:51,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/b7df0f6080024fb68324baefdc714dc5 is 50, key is test_row_0/B:col10/1732536230198/Put/seqid=0 2024-11-25T12:03:51,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741877_1053 (size=12151) 2024-11-25T12:03:51,353 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/b7df0f6080024fb68324baefdc714dc5 2024-11-25T12:03:51,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:51,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/fe37d773be3a4d94bc66dbbc5562b056 is 50, key is test_row_0/C:col10/1732536230198/Put/seqid=0 2024-11-25T12:03:51,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741878_1054 (size=12151) 2024-11-25T12:03:51,413 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/fe37d773be3a4d94bc66dbbc5562b056 2024-11-25T12:03:51,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536291401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:51,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:51,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536291401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:51,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536291407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:51,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536291410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/5cb5a1b237fc4016a3e002cd97635661 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/5cb5a1b237fc4016a3e002cd97635661 2024-11-25T12:03:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T12:03:51,434 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/5cb5a1b237fc4016a3e002cd97635661, entries=150, sequenceid=214, filesize=11.9 K 2024-11-25T12:03:51,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/b7df0f6080024fb68324baefdc714dc5 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/b7df0f6080024fb68324baefdc714dc5 2024-11-25T12:03:51,492 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/b7df0f6080024fb68324baefdc714dc5, entries=150, sequenceid=214, filesize=11.9 K 2024-11-25T12:03:51,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/fe37d773be3a4d94bc66dbbc5562b056 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/fe37d773be3a4d94bc66dbbc5562b056 2024-11-25T12:03:51,505 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/fe37d773be3a4d94bc66dbbc5562b056, entries=150, sequenceid=214, filesize=11.9 K 2024-11-25T12:03:51,507 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 057d2058e8441af178fff3bc63c12b69 in 239ms, sequenceid=214, compaction requested=true 2024-11-25T12:03:51,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:51,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:51,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-25T12:03:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-25T12:03:51,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-25T12:03:51,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 401 msec 2024-11-25T12:03:51,530 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 421 msec 2024-11-25T12:03:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:51,535 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:03:51,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-25T12:03:51,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/a7812e48946249f684011f396578aa5d is 50, key is test_row_0/A:col10/1732536231395/Put/seqid=0 2024-11-25T12:03:51,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741879_1055 (size=12151) 2024-11-25T12:03:51,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/a7812e48946249f684011f396578aa5d 2024-11-25T12:03:51,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/898a22bfd0854385838a19f0f68b389c is 50, key is test_row_0/B:col10/1732536231395/Put/seqid=0 2024-11-25T12:03:51,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741880_1056 (size=12151) 2024-11-25T12:03:51,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/898a22bfd0854385838a19f0f68b389c 2024-11-25T12:03:51,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536291631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:51,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536291633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:51,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536291639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:51,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536291642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:51,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/63f466bb74d644efba621b7b3272896c is 50, key is test_row_0/C:col10/1732536231395/Put/seqid=0 2024-11-25T12:03:51,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741881_1057 (size=12151) 2024-11-25T12:03:51,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/63f466bb74d644efba621b7b3272896c 2024-11-25T12:03:51,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/a7812e48946249f684011f396578aa5d as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/a7812e48946249f684011f396578aa5d 2024-11-25T12:03:51,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/a7812e48946249f684011f396578aa5d, entries=150, sequenceid=232, filesize=11.9 K 2024-11-25T12:03:51,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/898a22bfd0854385838a19f0f68b389c as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/898a22bfd0854385838a19f0f68b389c 2024-11-25T12:03:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-25T12:03:51,737 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:03:51,738 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:03:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-25T12:03:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T12:03:51,748 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:03:51,750 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:03:51,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:03:51,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536291753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:51,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536291756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:51,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/898a22bfd0854385838a19f0f68b389c, entries=150, sequenceid=232, filesize=11.9 K 2024-11-25T12:03:51,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/63f466bb74d644efba621b7b3272896c as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/63f466bb74d644efba621b7b3272896c 2024-11-25T12:03:51,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536291766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:51,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536291768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:51,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/63f466bb74d644efba621b7b3272896c, entries=150, sequenceid=232, filesize=11.9 K 2024-11-25T12:03:51,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:51,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:51,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 057d2058e8441af178fff3bc63c12b69 in 248ms, sequenceid=232, compaction requested=true 2024-11-25T12:03:51,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:51,783 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:03:51,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:03:51,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:51,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:03:51,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:03:51,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:03:51,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:03:51,786 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:03:51,786 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/A is initiating minor compaction (all files) 2024-11-25T12:03:51,786 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/A in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:51,786 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/caa9234026f24447a7b4b95afa20c9f9, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/b561ff7d029a4012b60ffade52ca4bf0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/5cb5a1b237fc4016a3e002cd97635661, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/a7812e48946249f684011f396578aa5d] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=47.8 K 2024-11-25T12:03:51,787 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting caa9234026f24447a7b4b95afa20c9f9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732536228584 2024-11-25T12:03:51,787 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting b561ff7d029a4012b60ffade52ca4bf0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732536228979 2024-11-25T12:03:51,788 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cb5a1b237fc4016a3e002cd97635661, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732536230189 2024-11-25T12:03:51,788 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:03:51,789 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting a7812e48946249f684011f396578aa5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732536231395 2024-11-25T12:03:51,797 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:03:51,797 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/B is initiating minor compaction (all files) 2024-11-25T12:03:51,797 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/B in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:51,797 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/bb0f252910ab42bcac0f275328c166f8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7bb1e4719f3b4fa9b9122ee81b97adc8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/b7df0f6080024fb68324baefdc714dc5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/898a22bfd0854385838a19f0f68b389c] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=47.8 K 2024-11-25T12:03:51,798 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb0f252910ab42bcac0f275328c166f8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732536228584 2024-11-25T12:03:51,799 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bb1e4719f3b4fa9b9122ee81b97adc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732536228979 2024-11-25T12:03:51,805 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7df0f6080024fb68324baefdc714dc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732536230189 2024-11-25T12:03:51,806 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 898a22bfd0854385838a19f0f68b389c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732536231395 2024-11-25T12:03:51,821 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#A#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:51,822 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/4d7b2c6634bb4677bdb2f5f2b52e7825 is 50, key is test_row_0/A:col10/1732536231395/Put/seqid=0 2024-11-25T12:03:51,831 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#B#compaction#46 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:51,833 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/7ba62eaebbf04edd83a98f7f911382f2 is 50, key is test_row_0/B:col10/1732536231395/Put/seqid=0 2024-11-25T12:03:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T12:03:51,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741882_1058 (size=12663) 2024-11-25T12:03:51,881 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/4d7b2c6634bb4677bdb2f5f2b52e7825 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/4d7b2c6634bb4677bdb2f5f2b52e7825 2024-11-25T12:03:51,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741883_1059 (size=12663) 2024-11-25T12:03:51,891 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/A of 057d2058e8441af178fff3bc63c12b69 into 4d7b2c6634bb4677bdb2f5f2b52e7825(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:51,891 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:51,891 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/A, priority=12, startTime=1732536231783; duration=0sec 2024-11-25T12:03:51,892 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:03:51,892 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:A 2024-11-25T12:03:51,892 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:03:51,898 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:03:51,898 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/C is initiating minor compaction (all files) 2024-11-25T12:03:51,898 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/C in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:51,900 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/54d99e1e87af471f8cb1f08328f45967, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/dce8846486fa47d78d89499186fa189b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/fe37d773be3a4d94bc66dbbc5562b056, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/63f466bb74d644efba621b7b3272896c] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=47.8 K 2024-11-25T12:03:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:51,905 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:03:51,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-25T12:03:51,911 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 54d99e1e87af471f8cb1f08328f45967, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732536228584 2024-11-25T12:03:51,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:03:51,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:51,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:51,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:51,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:51,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:51,917 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting dce8846486fa47d78d89499186fa189b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732536228979 2024-11-25T12:03:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:51,926 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting fe37d773be3a4d94bc66dbbc5562b056, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732536230189 2024-11-25T12:03:51,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/b270257e32624e72a392fcbb69c93ba8 is 50, key is test_row_0/A:col10/1732536231902/Put/seqid=0 2024-11-25T12:03:51,930 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 63f466bb74d644efba621b7b3272896c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732536231395 2024-11-25T12:03:51,931 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/7ba62eaebbf04edd83a98f7f911382f2 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7ba62eaebbf04edd83a98f7f911382f2 2024-11-25T12:03:51,950 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/B of 057d2058e8441af178fff3bc63c12b69 into 7ba62eaebbf04edd83a98f7f911382f2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:51,950 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:51,950 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/B, priority=12, startTime=1732536231785; duration=0sec 2024-11-25T12:03:51,951 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:51,951 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:B 2024-11-25T12:03:51,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741884_1060 (size=14541) 2024-11-25T12:03:51,967 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#C#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:51,968 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/1cd4aab66c7d4a368a708eee4e1dd847 is 50, key is test_row_0/C:col10/1732536231395/Put/seqid=0 2024-11-25T12:03:51,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/b270257e32624e72a392fcbb69c93ba8 2024-11-25T12:03:52,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741885_1061 (size=12663) 2024-11-25T12:03:52,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/719cae236cc948a99c4ae4eff05d78be is 50, key is test_row_0/B:col10/1732536231902/Put/seqid=0 2024-11-25T12:03:52,017 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/1cd4aab66c7d4a368a708eee4e1dd847 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/1cd4aab66c7d4a368a708eee4e1dd847 2024-11-25T12:03:52,027 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/C of 057d2058e8441af178fff3bc63c12b69 into 1cd4aab66c7d4a368a708eee4e1dd847(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:52,028 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:52,028 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/C, priority=12, startTime=1732536231785; duration=0sec 2024-11-25T12:03:52,028 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:52,028 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:C 2024-11-25T12:03:52,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536292027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:52,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:52,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536292027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:52,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:52,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:52,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536292030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:52,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:52,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536292031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:52,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:52,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536292044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:52,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:52,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741886_1062 (size=12151) 2024-11-25T12:03:52,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/719cae236cc948a99c4ae4eff05d78be 2024-11-25T12:03:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T12:03:52,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:03:52,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/c2ee3165ad6b45fc9e1f2ca0c7214b6d is 50, key is test_row_0/C:col10/1732536231902/Put/seqid=0 2024-11-25T12:03:52,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:52,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:52,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:52,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:52,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:52,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741887_1063 (size=12151) 2024-11-25T12:03:52,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/c2ee3165ad6b45fc9e1f2ca0c7214b6d 2024-11-25T12:03:52,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/b270257e32624e72a392fcbb69c93ba8 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/b270257e32624e72a392fcbb69c93ba8 2024-11-25T12:03:52,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/b270257e32624e72a392fcbb69c93ba8, entries=200, sequenceid=253, filesize=14.2 K 2024-11-25T12:03:52,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/719cae236cc948a99c4ae4eff05d78be as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/719cae236cc948a99c4ae4eff05d78be 2024-11-25T12:03:52,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/719cae236cc948a99c4ae4eff05d78be, entries=150, sequenceid=253, filesize=11.9 K 2024-11-25T12:03:52,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/c2ee3165ad6b45fc9e1f2ca0c7214b6d as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/c2ee3165ad6b45fc9e1f2ca0c7214b6d 2024-11-25T12:03:52,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536292157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:52,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:52,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536292160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:52,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:52,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/c2ee3165ad6b45fc9e1f2ca0c7214b6d, entries=150, sequenceid=253, filesize=11.9 K 2024-11-25T12:03:52,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 057d2058e8441af178fff3bc63c12b69 in 271ms, sequenceid=253, compaction requested=false 2024-11-25T12:03:52,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:52,181 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:03:52,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-25T12:03:52,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/e0d04186093642e09b3ca352b5b1db6b is 50, key is test_row_0/A:col10/1732536232007/Put/seqid=0 2024-11-25T12:03:52,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:03:52,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:52,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:52,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:52,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:52,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:52,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741888_1064 (size=12301) 2024-11-25T12:03:52,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/e0d04186093642e09b3ca352b5b1db6b 2024-11-25T12:03:52,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536292292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:52,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536292302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536292303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:52,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/301e5398a9b84863a6e7fc9b37c8f98f is 50, key is test_row_0/B:col10/1732536232007/Put/seqid=0 2024-11-25T12:03:52,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741889_1065 (size=12301) 2024-11-25T12:03:52,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:52,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536292377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536292377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T12:03:52,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:52,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:03:52,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:52,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:52,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:52,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:52,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:52,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536292407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:52,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:52,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536292427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:52,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:52,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536292427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:52,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:52,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:03:52,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:52,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:52,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:52,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:52,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:52,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:52,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536292627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:52,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:52,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536292646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:52,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536292646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:52,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:52,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536292686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:52,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536292686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:52,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:52,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:03:52,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:52,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:52,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:52,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:52,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:52,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:52,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/301e5398a9b84863a6e7fc9b37c8f98f 2024-11-25T12:03:52,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/2f624b0b614f425986826477f5942e51 is 50, key is test_row_0/C:col10/1732536232007/Put/seqid=0 2024-11-25T12:03:52,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741890_1066 (size=12301) 2024-11-25T12:03:52,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/2f624b0b614f425986826477f5942e51 2024-11-25T12:03:52,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/e0d04186093642e09b3ca352b5b1db6b as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/e0d04186093642e09b3ca352b5b1db6b 2024-11-25T12:03:52,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/e0d04186093642e09b3ca352b5b1db6b, entries=150, sequenceid=273, filesize=12.0 K 2024-11-25T12:03:52,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/301e5398a9b84863a6e7fc9b37c8f98f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/301e5398a9b84863a6e7fc9b37c8f98f 2024-11-25T12:03:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T12:03:52,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/301e5398a9b84863a6e7fc9b37c8f98f, entries=150, sequenceid=273, filesize=12.0 K 2024-11-25T12:03:52,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/2f624b0b614f425986826477f5942e51 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/2f624b0b614f425986826477f5942e51 2024-11-25T12:03:52,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:03:52,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:52,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:52,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:52,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:52,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:52,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/2f624b0b614f425986826477f5942e51, entries=150, sequenceid=273, filesize=12.0 K 2024-11-25T12:03:52,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 057d2058e8441af178fff3bc63c12b69 in 723ms, sequenceid=273, compaction requested=true 2024-11-25T12:03:52,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:52,905 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:03:52,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:03:52,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:52,905 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:03:52,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:03:52,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:52,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:03:52,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:03:52,907 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:03:52,907 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/A is initiating minor compaction (all files) 2024-11-25T12:03:52,907 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/A in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:52,907 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/4d7b2c6634bb4677bdb2f5f2b52e7825, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/b270257e32624e72a392fcbb69c93ba8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/e0d04186093642e09b3ca352b5b1db6b] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=38.6 K 2024-11-25T12:03:52,908 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:03:52,908 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/B is initiating minor compaction (all files) 2024-11-25T12:03:52,908 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/B in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:52,908 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7ba62eaebbf04edd83a98f7f911382f2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/719cae236cc948a99c4ae4eff05d78be, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/301e5398a9b84863a6e7fc9b37c8f98f] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=36.2 K 2024-11-25T12:03:52,908 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d7b2c6634bb4677bdb2f5f2b52e7825, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732536231395 2024-11-25T12:03:52,909 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ba62eaebbf04edd83a98f7f911382f2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732536231395 2024-11-25T12:03:52,909 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting b270257e32624e72a392fcbb69c93ba8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732536231611 2024-11-25T12:03:52,910 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 719cae236cc948a99c4ae4eff05d78be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732536231611 2024-11-25T12:03:52,910 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0d04186093642e09b3ca352b5b1db6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732536232007 2024-11-25T12:03:52,911 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 301e5398a9b84863a6e7fc9b37c8f98f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732536232007 2024-11-25T12:03:52,936 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#A#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:52,937 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/04f3f39b9fe64717bdefa742d7c1e103 is 50, key is test_row_0/A:col10/1732536232007/Put/seqid=0 2024-11-25T12:03:52,939 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:52,940 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/e81881fd6c93489ababa5e8994366405 is 50, key is test_row_0/B:col10/1732536232007/Put/seqid=0 2024-11-25T12:03:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:52,945 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:03:52,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-25T12:03:52,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/0cbbe4ca41f24c8697a1c6f3969f2b7a is 50, key is test_row_0/A:col10/1732536232288/Put/seqid=0 2024-11-25T12:03:52,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741891_1067 (size=12915) 2024-11-25T12:03:52,999 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/04f3f39b9fe64717bdefa742d7c1e103 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/04f3f39b9fe64717bdefa742d7c1e103 2024-11-25T12:03:53,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741892_1068 (size=12915) 2024-11-25T12:03:53,009 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/A of 057d2058e8441af178fff3bc63c12b69 into 04f3f39b9fe64717bdefa742d7c1e103(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:53,009 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:53,009 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/A, priority=13, startTime=1732536232905; duration=0sec 2024-11-25T12:03:53,009 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:03:53,010 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:A 2024-11-25T12:03:53,010 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:03:53,015 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:03:53,015 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/C is initiating minor compaction (all files) 2024-11-25T12:03:53,015 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/C in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:53,015 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/1cd4aab66c7d4a368a708eee4e1dd847, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/c2ee3165ad6b45fc9e1f2ca0c7214b6d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/2f624b0b614f425986826477f5942e51] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=36.2 K 2024-11-25T12:03:53,016 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cd4aab66c7d4a368a708eee4e1dd847, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732536231395 2024-11-25T12:03:53,017 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/e81881fd6c93489ababa5e8994366405 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/e81881fd6c93489ababa5e8994366405 2024-11-25T12:03:53,018 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2ee3165ad6b45fc9e1f2ca0c7214b6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732536231611 2024-11-25T12:03:53,019 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f624b0b614f425986826477f5942e51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732536232007 2024-11-25T12:03:53,035 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/B of 057d2058e8441af178fff3bc63c12b69 into e81881fd6c93489ababa5e8994366405(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:53,035 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:53,035 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/B, priority=13, startTime=1732536232905; duration=0sec 2024-11-25T12:03:53,035 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:53,035 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:B 2024-11-25T12:03:53,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:53,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536293014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:53,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:53,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536293028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:53,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:53,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536293031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:53,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:53,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741893_1069 (size=12301) 2024-11-25T12:03:53,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/0cbbe4ca41f24c8697a1c6f3969f2b7a 2024-11-25T12:03:53,048 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#C#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:53,049 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/7aa9400864a345c086bf0c6ba35202cd is 50, key is test_row_0/C:col10/1732536232007/Put/seqid=0 2024-11-25T12:03:53,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:03:53,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:53,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:53,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:53,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:53,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:53,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:53,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/d8c968509e804ae89f8e7773ed6552b8 is 50, key is test_row_0/B:col10/1732536232288/Put/seqid=0 2024-11-25T12:03:53,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741894_1070 (size=12915) 2024-11-25T12:03:53,111 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/7aa9400864a345c086bf0c6ba35202cd as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7aa9400864a345c086bf0c6ba35202cd 2024-11-25T12:03:53,134 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/C of 057d2058e8441af178fff3bc63c12b69 into 7aa9400864a345c086bf0c6ba35202cd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:53,134 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:53,134 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/C, priority=13, startTime=1732536232906; duration=0sec 2024-11-25T12:03:53,134 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:53,134 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:C 2024-11-25T12:03:53,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741895_1071 (size=12301) 2024-11-25T12:03:53,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:53,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/d8c968509e804ae89f8e7773ed6552b8 2024-11-25T12:03:53,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:53,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536293153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:53,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:53,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536293165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:53,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:53,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536293154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:53,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:53,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/72ad8166874e4e8cacf860ae0c4b8a95 is 50, key is test_row_0/C:col10/1732536232288/Put/seqid=0 2024-11-25T12:03:53,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:53,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536293196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:53,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:03:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:53,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:53,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:53,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:53,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:53,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:53,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:53,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:53,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536293206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:53,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:53,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741896_1072 (size=12301) 2024-11-25T12:03:53,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:03:53,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:53,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:53,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:53,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:53,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:53,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:53,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536293376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:53,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:53,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536293389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:53,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:53,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536293389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:53,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:03:53,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:53,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:53,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:53,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:53,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:53,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:53,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/72ad8166874e4e8cacf860ae0c4b8a95 2024-11-25T12:03:53,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/0cbbe4ca41f24c8697a1c6f3969f2b7a as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/0cbbe4ca41f24c8697a1c6f3969f2b7a 2024-11-25T12:03:53,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/0cbbe4ca41f24c8697a1c6f3969f2b7a, entries=150, sequenceid=293, filesize=12.0 K 2024-11-25T12:03:53,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/d8c968509e804ae89f8e7773ed6552b8 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/d8c968509e804ae89f8e7773ed6552b8 2024-11-25T12:03:53,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:03:53,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:53,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:53,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:53,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:53,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:53,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/d8c968509e804ae89f8e7773ed6552b8, entries=150, sequenceid=293, filesize=12.0 K 2024-11-25T12:03:53,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/72ad8166874e4e8cacf860ae0c4b8a95 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/72ad8166874e4e8cacf860ae0c4b8a95 2024-11-25T12:03:53,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:53,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536293699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536293699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:53,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:53,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536293706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:53,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:53,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:53,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/72ad8166874e4e8cacf860ae0c4b8a95, entries=150, sequenceid=293, filesize=12.0 K 2024-11-25T12:03:53,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 057d2058e8441af178fff3bc63c12b69 in 769ms, sequenceid=293, compaction requested=false 2024-11-25T12:03:53,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:53,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-25T12:03:53,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:53,844 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-25T12:03:53,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/c2a91bd2a0e54a77abd21f71033030da is 50, key is test_row_0/A:col10/1732536233024/Put/seqid=0 2024-11-25T12:03:53,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T12:03:53,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741897_1073 (size=12301) 2024-11-25T12:03:53,901 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/c2a91bd2a0e54a77abd21f71033030da 2024-11-25T12:03:53,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/8a361dabd41347f7a51beaea3a881126 is 50, key is test_row_0/B:col10/1732536233024/Put/seqid=0 2024-11-25T12:03:53,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741898_1074 (size=12301) 2024-11-25T12:03:53,982 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/8a361dabd41347f7a51beaea3a881126 2024-11-25T12:03:54,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/194d532317b94621abb3a08ebddd650e is 50, key is test_row_0/C:col10/1732536233024/Put/seqid=0 2024-11-25T12:03:54,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741899_1075 (size=12301) 2024-11-25T12:03:54,053 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/194d532317b94621abb3a08ebddd650e 2024-11-25T12:03:54,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/c2a91bd2a0e54a77abd21f71033030da as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/c2a91bd2a0e54a77abd21f71033030da 2024-11-25T12:03:54,082 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/c2a91bd2a0e54a77abd21f71033030da, entries=150, sequenceid=312, filesize=12.0 K 2024-11-25T12:03:54,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/8a361dabd41347f7a51beaea3a881126 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/8a361dabd41347f7a51beaea3a881126 2024-11-25T12:03:54,100 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/8a361dabd41347f7a51beaea3a881126, entries=150, sequenceid=312, filesize=12.0 K 2024-11-25T12:03:54,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/194d532317b94621abb3a08ebddd650e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/194d532317b94621abb3a08ebddd650e 2024-11-25T12:03:54,109 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/194d532317b94621abb3a08ebddd650e, entries=150, sequenceid=312, filesize=12.0 K 2024-11-25T12:03:54,117 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=0 B/0 for 057d2058e8441af178fff3bc63c12b69 in 274ms, sequenceid=312, compaction requested=true 2024-11-25T12:03:54,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:54,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:54,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-25T12:03:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-25T12:03:54,130 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-25T12:03:54,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3740 sec 2024-11-25T12:03:54,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.3930 sec 2024-11-25T12:03:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:54,267 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:03:54,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:03:54,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/5efc0c396a194a95a80d19a2a0ae161e is 50, key is test_row_0/A:col10/1732536234265/Put/seqid=0 2024-11-25T12:03:54,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741900_1076 (size=12301) 2024-11-25T12:03:54,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/5efc0c396a194a95a80d19a2a0ae161e 2024-11-25T12:03:54,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/027c3970774c4453af21c253e8111fa4 is 50, key is test_row_0/B:col10/1732536234265/Put/seqid=0 2024-11-25T12:03:54,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741901_1077 (size=12301) 2024-11-25T12:03:54,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/027c3970774c4453af21c253e8111fa4 2024-11-25T12:03:54,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536294413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:54,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:54,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536294423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:54,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/7326fb52e8a54faf8548ce61c9da828a is 50, key is test_row_0/C:col10/1732536234265/Put/seqid=0 2024-11-25T12:03:54,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:54,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536294432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:54,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536294434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:54,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:54,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536294434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:54,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:54,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:54,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741902_1078 (size=12301) 2024-11-25T12:03:54,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/7326fb52e8a54faf8548ce61c9da828a 2024-11-25T12:03:54,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/5efc0c396a194a95a80d19a2a0ae161e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/5efc0c396a194a95a80d19a2a0ae161e 2024-11-25T12:03:54,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/5efc0c396a194a95a80d19a2a0ae161e, entries=150, sequenceid=325, filesize=12.0 K 2024-11-25T12:03:54,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/027c3970774c4453af21c253e8111fa4 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/027c3970774c4453af21c253e8111fa4 2024-11-25T12:03:54,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/027c3970774c4453af21c253e8111fa4, entries=150, sequenceid=325, filesize=12.0 K 2024-11-25T12:03:54,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/7326fb52e8a54faf8548ce61c9da828a as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7326fb52e8a54faf8548ce61c9da828a 2024-11-25T12:03:54,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7326fb52e8a54faf8548ce61c9da828a, entries=150, sequenceid=325, filesize=12.0 K 2024-11-25T12:03:54,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 057d2058e8441af178fff3bc63c12b69 in 248ms, sequenceid=325, compaction requested=true 2024-11-25T12:03:54,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:54,517 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:03:54,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:03:54,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:54,518 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:03:54,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:03:54,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:54,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:03:54,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:03:54,520 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:03:54,520 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/A is initiating minor compaction (all files) 2024-11-25T12:03:54,520 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:03:54,520 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/A in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:54,520 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/B is initiating minor compaction (all files) 2024-11-25T12:03:54,520 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/04f3f39b9fe64717bdefa742d7c1e103, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/0cbbe4ca41f24c8697a1c6f3969f2b7a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/c2a91bd2a0e54a77abd21f71033030da, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/5efc0c396a194a95a80d19a2a0ae161e] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=48.7 K 2024-11-25T12:03:54,520 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/B in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:54,520 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/e81881fd6c93489ababa5e8994366405, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/d8c968509e804ae89f8e7773ed6552b8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/8a361dabd41347f7a51beaea3a881126, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/027c3970774c4453af21c253e8111fa4] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=48.7 K 2024-11-25T12:03:54,521 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04f3f39b9fe64717bdefa742d7c1e103, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732536232007 2024-11-25T12:03:54,521 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cbbe4ca41f24c8697a1c6f3969f2b7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732536232288 2024-11-25T12:03:54,521 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting e81881fd6c93489ababa5e8994366405, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732536232007 2024-11-25T12:03:54,523 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting d8c968509e804ae89f8e7773ed6552b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732536232288 2024-11-25T12:03:54,523 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2a91bd2a0e54a77abd21f71033030da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732536232985 2024-11-25T12:03:54,525 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5efc0c396a194a95a80d19a2a0ae161e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732536234254 2024-11-25T12:03:54,525 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a361dabd41347f7a51beaea3a881126, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732536232985 2024-11-25T12:03:54,526 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 027c3970774c4453af21c253e8111fa4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732536234254 2024-11-25T12:03:54,548 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#A#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:54,548 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/a2412afb703e4b7ba90a28dca55f7910 is 50, key is test_row_0/A:col10/1732536234265/Put/seqid=0 2024-11-25T12:03:54,552 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#B#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:54,552 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/41d83c31506e41b4b593d0fbfba36d17 is 50, key is test_row_0/B:col10/1732536234265/Put/seqid=0 2024-11-25T12:03:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:54,554 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:03:54,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T12:03:54,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536294588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:54,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536294596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:54,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536294598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:54,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536294590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:54,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536294614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:54,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741903_1079 (size=13051) 2024-11-25T12:03:54,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/31363a02e59f4d73a8eada9bca2ab247 is 50, key is test_row_0/A:col10/1732536234542/Put/seqid=0 2024-11-25T12:03:54,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741904_1080 (size=13051) 2024-11-25T12:03:54,665 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/a2412afb703e4b7ba90a28dca55f7910 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/a2412afb703e4b7ba90a28dca55f7910 2024-11-25T12:03:54,676 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/A of 057d2058e8441af178fff3bc63c12b69 into a2412afb703e4b7ba90a28dca55f7910(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:54,676 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:54,676 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/A, priority=12, startTime=1732536234517; duration=0sec 2024-11-25T12:03:54,676 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:03:54,676 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:A 2024-11-25T12:03:54,676 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:03:54,683 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:03:54,683 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/C is initiating minor compaction (all files) 2024-11-25T12:03:54,683 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/C in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:54,683 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7aa9400864a345c086bf0c6ba35202cd, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/72ad8166874e4e8cacf860ae0c4b8a95, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/194d532317b94621abb3a08ebddd650e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7326fb52e8a54faf8548ce61c9da828a] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=48.7 K 2024-11-25T12:03:54,684 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aa9400864a345c086bf0c6ba35202cd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732536232007 2024-11-25T12:03:54,685 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72ad8166874e4e8cacf860ae0c4b8a95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732536232288 2024-11-25T12:03:54,685 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 194d532317b94621abb3a08ebddd650e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1732536232985 2024-11-25T12:03:54,686 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7326fb52e8a54faf8548ce61c9da828a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732536234254 2024-11-25T12:03:54,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741905_1081 (size=14741) 2024-11-25T12:03:54,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536294719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:54,728 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#C#compaction#69 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:54,729 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/625f140072784fa9994dd8ec6ee14b62 is 50, key is test_row_0/C:col10/1732536234265/Put/seqid=0 2024-11-25T12:03:54,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:54,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536294726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:54,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536294726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:54,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:54,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536294726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:54,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536294730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:54,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741906_1082 (size=13051) 2024-11-25T12:03:54,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536294937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:54,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536294948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:54,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:54,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536294948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:54,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536294948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:54,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:54,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536294949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:54,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:55,072 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/41d83c31506e41b4b593d0fbfba36d17 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/41d83c31506e41b4b593d0fbfba36d17 2024-11-25T12:03:55,093 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/B of 057d2058e8441af178fff3bc63c12b69 into 41d83c31506e41b4b593d0fbfba36d17(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:55,093 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:55,093 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/B, priority=12, startTime=1732536234518; duration=0sec 2024-11-25T12:03:55,093 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:55,093 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:B 2024-11-25T12:03:55,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/31363a02e59f4d73a8eada9bca2ab247 2024-11-25T12:03:55,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/180f82cce61246c9acf76dc068f96778 is 50, key is test_row_0/B:col10/1732536234542/Put/seqid=0 2024-11-25T12:03:55,188 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/625f140072784fa9994dd8ec6ee14b62 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/625f140072784fa9994dd8ec6ee14b62 2024-11-25T12:03:55,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741907_1083 (size=12301) 2024-11-25T12:03:55,228 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/C of 057d2058e8441af178fff3bc63c12b69 into 625f140072784fa9994dd8ec6ee14b62(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:55,229 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:55,229 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/C, priority=12, startTime=1732536234518; duration=0sec 2024-11-25T12:03:55,229 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:55,229 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:C 2024-11-25T12:03:55,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:55,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536295267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:55,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:55,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:55,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:55,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:55,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536295267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:55,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:55,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:55,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:55,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:55,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:55,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536295267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:55,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:55,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:55,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:55,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:55,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536295269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:55,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536295267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:55,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:55,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:55,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:55,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:55,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/180f82cce61246c9acf76dc068f96778 2024-11-25T12:03:55,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/b4c416c687a74efb82113030b52fd50e is 50, key is test_row_0/C:col10/1732536234542/Put/seqid=0 2024-11-25T12:03:55,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741908_1084 (size=12301) 2024-11-25T12:03:55,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/b4c416c687a74efb82113030b52fd50e 2024-11-25T12:03:55,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/31363a02e59f4d73a8eada9bca2ab247 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/31363a02e59f4d73a8eada9bca2ab247 2024-11-25T12:03:55,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536295781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:55,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:55,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:55,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:55,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536295781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:55,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536295778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:55,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536295781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:55,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:55,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:55,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:55,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536295786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:55,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/31363a02e59f4d73a8eada9bca2ab247, entries=200, sequenceid=349, filesize=14.4 K 2024-11-25T12:03:55,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/180f82cce61246c9acf76dc068f96778 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/180f82cce61246c9acf76dc068f96778 2024-11-25T12:03:55,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/180f82cce61246c9acf76dc068f96778, entries=150, sequenceid=349, filesize=12.0 K 2024-11-25T12:03:55,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/b4c416c687a74efb82113030b52fd50e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/b4c416c687a74efb82113030b52fd50e 2024-11-25T12:03:55,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/b4c416c687a74efb82113030b52fd50e, entries=150, sequenceid=349, filesize=12.0 K 2024-11-25T12:03:55,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 057d2058e8441af178fff3bc63c12b69 in 1338ms, sequenceid=349, compaction requested=false 2024-11-25T12:03:55,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-25T12:03:55,928 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:03:55,929 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:03:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-25T12:03:55,934 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:03:55,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T12:03:55,940 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:03:55,941 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:03:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T12:03:56,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-25T12:03:56,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:56,109 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T12:03:56,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/ccff0d972cbf41bab54b41dde0fcacb3 is 50, key is test_row_0/A:col10/1732536234595/Put/seqid=0 2024-11-25T12:03:56,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741909_1085 (size=12301) 2024-11-25T12:03:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T12:03:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T12:03:56,631 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/ccff0d972cbf41bab54b41dde0fcacb3 2024-11-25T12:03:56,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/7ff7ca3ccc00422bacd0f6f8317044c4 is 50, key is test_row_0/B:col10/1732536234595/Put/seqid=0 2024-11-25T12:03:56,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741910_1086 (size=12301) 2024-11-25T12:03:56,718 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/7ff7ca3ccc00422bacd0f6f8317044c4 2024-11-25T12:03:56,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/7b8d4ee868b449ffb5402b6dd96083a6 is 50, key is test_row_0/C:col10/1732536234595/Put/seqid=0 2024-11-25T12:03:56,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:56,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741911_1087 (size=12301) 2024-11-25T12:03:56,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:56,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536296865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:56,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:56,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:56,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:56,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536296868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:56,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536296868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:56,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536296870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:56,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536296870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:56,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:56,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536296986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536296987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:56,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536296987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:56,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:56,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:56,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:56,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:56,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:56,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536297004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536297004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T12:03:57,203 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/7b8d4ee868b449ffb5402b6dd96083a6 2024-11-25T12:03:57,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/ccff0d972cbf41bab54b41dde0fcacb3 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/ccff0d972cbf41bab54b41dde0fcacb3 2024-11-25T12:03:57,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536297209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536297218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536297209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,225 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/ccff0d972cbf41bab54b41dde0fcacb3, entries=150, sequenceid=364, filesize=12.0 K 2024-11-25T12:03:57,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536297218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/7ff7ca3ccc00422bacd0f6f8317044c4 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7ff7ca3ccc00422bacd0f6f8317044c4 2024-11-25T12:03:57,240 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7ff7ca3ccc00422bacd0f6f8317044c4, entries=150, sequenceid=364, filesize=12.0 K 2024-11-25T12:03:57,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536297218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/7b8d4ee868b449ffb5402b6dd96083a6 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7b8d4ee868b449ffb5402b6dd96083a6 2024-11-25T12:03:57,269 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7b8d4ee868b449ffb5402b6dd96083a6, entries=150, sequenceid=364, filesize=12.0 K 2024-11-25T12:03:57,271 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 057d2058e8441af178fff3bc63c12b69 in 1161ms, sequenceid=364, compaction requested=true 2024-11-25T12:03:57,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:57,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:57,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-25T12:03:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-25T12:03:57,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-25T12:03:57,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3350 sec 2024-11-25T12:03:57,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.3860 sec 2024-11-25T12:03:57,543 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:03:57,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T12:03:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:57,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/0546aa67b7344e99b6009e1dde88b07a is 50, key is test_row_0/A:col10/1732536237541/Put/seqid=0 2024-11-25T12:03:57,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536297564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536297566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536297571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536297575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536297578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741912_1088 (size=14741) 2024-11-25T12:03:57,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536297687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536297688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536297689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536297689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536297689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536297896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536297897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536297907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536297908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:57,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536297909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:57,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:57,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/0546aa67b7344e99b6009e1dde88b07a 2024-11-25T12:03:58,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/116b6919129f4a0a98df6fe2a8741bd0 is 50, key is test_row_0/B:col10/1732536237541/Put/seqid=0 2024-11-25T12:03:58,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741913_1089 (size=12301) 2024-11-25T12:03:58,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/116b6919129f4a0a98df6fe2a8741bd0 2024-11-25T12:03:58,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T12:03:58,098 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:03:58,098 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:03:58,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-25T12:03:58,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T12:03:58,121 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:03:58,138 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:03:58,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:03:58,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/c61bb1ad28c44268bc7089a5b4b7e9e2 is 50, key is test_row_0/C:col10/1732536237541/Put/seqid=0 2024-11-25T12:03:58,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:58,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536298207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:58,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741914_1090 (size=12301) 2024-11-25T12:03:58,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:58,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536298207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:58,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:58,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/c61bb1ad28c44268bc7089a5b4b7e9e2 2024-11-25T12:03:58,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:58,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536298216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T12:03:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:58,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536298227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:58,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:58,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536298233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:58,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:58,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:58,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/0546aa67b7344e99b6009e1dde88b07a as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/0546aa67b7344e99b6009e1dde88b07a 2024-11-25T12:03:58,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-25T12:03:58,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:58,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:58,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:58,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:58,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:58,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/0546aa67b7344e99b6009e1dde88b07a, entries=200, sequenceid=389, filesize=14.4 K 2024-11-25T12:03:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:03:58,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/116b6919129f4a0a98df6fe2a8741bd0 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/116b6919129f4a0a98df6fe2a8741bd0 2024-11-25T12:03:58,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/116b6919129f4a0a98df6fe2a8741bd0, entries=150, sequenceid=389, filesize=12.0 K 2024-11-25T12:03:58,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/c61bb1ad28c44268bc7089a5b4b7e9e2 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/c61bb1ad28c44268bc7089a5b4b7e9e2 2024-11-25T12:03:58,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/c61bb1ad28c44268bc7089a5b4b7e9e2, entries=150, sequenceid=389, filesize=12.0 K 2024-11-25T12:03:58,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 057d2058e8441af178fff3bc63c12b69 in 854ms, sequenceid=389, compaction requested=true 2024-11-25T12:03:58,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:58,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:03:58,397 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:58,397 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:03:58,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:03:58,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:03:58,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:03:58,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:03:58,398 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:03:58,410 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54834 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:03:58,410 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/A is initiating minor compaction (all files) 2024-11-25T12:03:58,410 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/A in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:58,410 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/a2412afb703e4b7ba90a28dca55f7910, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/31363a02e59f4d73a8eada9bca2ab247, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/ccff0d972cbf41bab54b41dde0fcacb3, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/0546aa67b7344e99b6009e1dde88b07a] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=53.5 K 2024-11-25T12:03:58,410 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:03:58,411 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/B is initiating minor compaction (all files) 2024-11-25T12:03:58,411 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/B in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:58,411 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/41d83c31506e41b4b593d0fbfba36d17, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/180f82cce61246c9acf76dc068f96778, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7ff7ca3ccc00422bacd0f6f8317044c4, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/116b6919129f4a0a98df6fe2a8741bd0] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=48.8 K 2024-11-25T12:03:58,411 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 41d83c31506e41b4b593d0fbfba36d17, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732536234254 2024-11-25T12:03:58,412 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2412afb703e4b7ba90a28dca55f7910, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732536234254 2024-11-25T12:03:58,412 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 180f82cce61246c9acf76dc068f96778, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732536234422 2024-11-25T12:03:58,413 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ff7ca3ccc00422bacd0f6f8317044c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732536234581 2024-11-25T12:03:58,416 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31363a02e59f4d73a8eada9bca2ab247, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732536234404 2024-11-25T12:03:58,416 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 116b6919129f4a0a98df6fe2a8741bd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732536236867 2024-11-25T12:03:58,416 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccff0d972cbf41bab54b41dde0fcacb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732536234581 2024-11-25T12:03:58,421 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0546aa67b7344e99b6009e1dde88b07a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732536236845 2024-11-25T12:03:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T12:03:58,460 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#B#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:58,461 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/10fa1b5a122446038e89209e76fc4453 is 50, key is test_row_0/B:col10/1732536237541/Put/seqid=0 2024-11-25T12:03:58,466 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#A#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:58,467 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/3de878cba7304cd7af91c69ae0332600 is 50, key is test_row_0/A:col10/1732536237541/Put/seqid=0 2024-11-25T12:03:58,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-25T12:03:58,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:58,477 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:03:58,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741915_1091 (size=13187) 2024-11-25T12:03:58,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741916_1092 (size=13187) 2024-11-25T12:03:58,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/9ea15456c4c542e99fed36eb66cc9516 is 50, key is test_row_0/A:col10/1732536237563/Put/seqid=0 2024-11-25T12:03:58,519 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/3de878cba7304cd7af91c69ae0332600 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/3de878cba7304cd7af91c69ae0332600 2024-11-25T12:03:58,520 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/10fa1b5a122446038e89209e76fc4453 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/10fa1b5a122446038e89209e76fc4453 2024-11-25T12:03:58,545 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/B of 057d2058e8441af178fff3bc63c12b69 into 10fa1b5a122446038e89209e76fc4453(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:58,545 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:58,546 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/B, priority=12, startTime=1732536238398; duration=0sec 2024-11-25T12:03:58,546 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:03:58,546 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:B 2024-11-25T12:03:58,546 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:03:58,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741917_1093 (size=12301) 2024-11-25T12:03:58,548 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:03:58,548 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/C is initiating minor compaction (all files) 2024-11-25T12:03:58,549 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/C in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:58,549 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/625f140072784fa9994dd8ec6ee14b62, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/b4c416c687a74efb82113030b52fd50e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7b8d4ee868b449ffb5402b6dd96083a6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/c61bb1ad28c44268bc7089a5b4b7e9e2] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=48.8 K 2024-11-25T12:03:58,550 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/A of 057d2058e8441af178fff3bc63c12b69 into 3de878cba7304cd7af91c69ae0332600(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:58,550 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:58,550 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/A, priority=12, startTime=1732536238397; duration=0sec 2024-11-25T12:03:58,550 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:58,550 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:A 2024-11-25T12:03:58,551 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/9ea15456c4c542e99fed36eb66cc9516 2024-11-25T12:03:58,551 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 625f140072784fa9994dd8ec6ee14b62, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732536234254 2024-11-25T12:03:58,552 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting b4c416c687a74efb82113030b52fd50e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732536234422 2024-11-25T12:03:58,553 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b8d4ee868b449ffb5402b6dd96083a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732536234581 2024-11-25T12:03:58,554 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting c61bb1ad28c44268bc7089a5b4b7e9e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732536236867 2024-11-25T12:03:58,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/f36067ea3b5f4c1c8b0709aa96853c20 is 50, key is test_row_0/B:col10/1732536237563/Put/seqid=0 2024-11-25T12:03:58,577 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#C#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:03:58,578 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/ad476b63d9664fe2a18a327ce389fa48 is 50, key is test_row_0/C:col10/1732536237541/Put/seqid=0 2024-11-25T12:03:58,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741919_1095 (size=13187) 2024-11-25T12:03:58,618 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/ad476b63d9664fe2a18a327ce389fa48 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/ad476b63d9664fe2a18a327ce389fa48 2024-11-25T12:03:58,627 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/C of 057d2058e8441af178fff3bc63c12b69 into ad476b63d9664fe2a18a327ce389fa48(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:03:58,627 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:58,627 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/C, priority=12, startTime=1732536238398; duration=0sec 2024-11-25T12:03:58,627 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:03:58,627 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:C 2024-11-25T12:03:58,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741918_1094 (size=12301) 2024-11-25T12:03:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:58,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:03:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T12:03:58,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:58,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536298784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:58,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536298786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:58,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:58,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536298789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:58,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536298789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:58,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536298790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:58,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:58,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:58,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536298896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:58,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536298896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:58,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536298908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:58,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536298908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:58,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:58,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536298908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:58,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:58,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:59,037 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/f36067ea3b5f4c1c8b0709aa96853c20 2024-11-25T12:03:59,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/3600a4686e894df9ad33c5d3a1ec4c0d is 50, key is test_row_0/C:col10/1732536237563/Put/seqid=0 2024-11-25T12:03:59,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:59,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536299117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:59,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:59,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536299117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:59,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:59,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:59,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536299126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:59,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:59,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741920_1096 (size=12301) 2024-11-25T12:03:59,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:59,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536299137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:59,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536299137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T12:03:59,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536299427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:59,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536299428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:59,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:59,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:59,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536299437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:59,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:59,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536299446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:59,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536299446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:59,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:59,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:59,552 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/3600a4686e894df9ad33c5d3a1ec4c0d 2024-11-25T12:03:59,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/9ea15456c4c542e99fed36eb66cc9516 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/9ea15456c4c542e99fed36eb66cc9516 2024-11-25T12:03:59,626 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/9ea15456c4c542e99fed36eb66cc9516, entries=150, sequenceid=400, filesize=12.0 K 2024-11-25T12:03:59,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/f36067ea3b5f4c1c8b0709aa96853c20 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/f36067ea3b5f4c1c8b0709aa96853c20 2024-11-25T12:03:59,664 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/f36067ea3b5f4c1c8b0709aa96853c20, entries=150, sequenceid=400, filesize=12.0 K 2024-11-25T12:03:59,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/3600a4686e894df9ad33c5d3a1ec4c0d as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/3600a4686e894df9ad33c5d3a1ec4c0d 2024-11-25T12:03:59,710 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/3600a4686e894df9ad33c5d3a1ec4c0d, entries=150, sequenceid=400, filesize=12.0 K 2024-11-25T12:03:59,727 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 057d2058e8441af178fff3bc63c12b69 in 1250ms, sequenceid=400, compaction requested=false 2024-11-25T12:03:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:03:59,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:03:59,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-25T12:03:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-25T12:03:59,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-25T12:03:59,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6030 sec 2024-11-25T12:03:59,773 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.6590 sec 2024-11-25T12:03:59,954 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:03:59,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:03:59,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-25T12:03:59,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/b6e99b8a4e9348b28cb6a3efaa664a10 is 50, key is test_row_0/A:col10/1732536238786/Put/seqid=0 2024-11-25T12:03:59,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536299967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:59,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38660 deadline: 1732536299968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:59,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:59,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:59,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536299973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:59,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536299974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:59,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741921_1097 (size=12301) 2024-11-25T12:03:59,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/b6e99b8a4e9348b28cb6a3efaa664a10 2024-11-25T12:03:59,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:03:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536299984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:03:59,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:03:59,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:03:59,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/9c6244dd4cc643a1b12fbcdd09547ee5 is 50, key is test_row_0/B:col10/1732536238786/Put/seqid=0 2024-11-25T12:04:00,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741922_1098 (size=12301) 2024-11-25T12:04:00,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/9c6244dd4cc643a1b12fbcdd09547ee5 2024-11-25T12:04:00,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/7627df3b7cf74e118ae63a66e42ab8db is 50, key is test_row_0/C:col10/1732536238786/Put/seqid=0 2024-11-25T12:04:00,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38628 deadline: 1732536300086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:00,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:00,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:00,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:00,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38652 deadline: 1732536300086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:00,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:00,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:00,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:00,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741923_1099 (size=12301) 2024-11-25T12:04:00,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/7627df3b7cf74e118ae63a66e42ab8db 2024-11-25T12:04:00,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38670 deadline: 1732536300089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:00,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/b6e99b8a4e9348b28cb6a3efaa664a10 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/b6e99b8a4e9348b28cb6a3efaa664a10 2024-11-25T12:04:00,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:00,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38676 deadline: 1732536300096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:00,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:00,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:00,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:00,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/b6e99b8a4e9348b28cb6a3efaa664a10, entries=150, sequenceid=430, filesize=12.0 K 2024-11-25T12:04:00,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/9c6244dd4cc643a1b12fbcdd09547ee5 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/9c6244dd4cc643a1b12fbcdd09547ee5 2024-11-25T12:04:00,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/9c6244dd4cc643a1b12fbcdd09547ee5, entries=150, sequenceid=430, filesize=12.0 K 2024-11-25T12:04:00,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/7627df3b7cf74e118ae63a66e42ab8db as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7627df3b7cf74e118ae63a66e42ab8db 2024-11-25T12:04:00,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7627df3b7cf74e118ae63a66e42ab8db, entries=150, sequenceid=430, filesize=12.0 K 2024-11-25T12:04:00,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 057d2058e8441af178fff3bc63c12b69 in 184ms, sequenceid=430, compaction requested=true 2024-11-25T12:04:00,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:04:00,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:04:00,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:00,139 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:00,139 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:00,140 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:00,140 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/B is initiating minor compaction (all files) 2024-11-25T12:04:00,140 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/B in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:00,141 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/10fa1b5a122446038e89209e76fc4453, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/f36067ea3b5f4c1c8b0709aa96853c20, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/9c6244dd4cc643a1b12fbcdd09547ee5] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=36.9 K 2024-11-25T12:04:00,141 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 10fa1b5a122446038e89209e76fc4453, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732536236867 2024-11-25T12:04:00,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:04:00,142 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:00,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:00,142 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/A is initiating minor compaction (all files) 2024-11-25T12:04:00,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 057d2058e8441af178fff3bc63c12b69:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:04:00,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:00,142 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/A in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:00,142 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/3de878cba7304cd7af91c69ae0332600, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/9ea15456c4c542e99fed36eb66cc9516, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/b6e99b8a4e9348b28cb6a3efaa664a10] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=36.9 K 2024-11-25T12:04:00,142 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting f36067ea3b5f4c1c8b0709aa96853c20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732536237549 2024-11-25T12:04:00,142 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3de878cba7304cd7af91c69ae0332600, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732536236867 2024-11-25T12:04:00,143 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c6244dd4cc643a1b12fbcdd09547ee5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732536238786 2024-11-25T12:04:00,143 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ea15456c4c542e99fed36eb66cc9516, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732536237549 2024-11-25T12:04:00,144 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6e99b8a4e9348b28cb6a3efaa664a10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732536238786 2024-11-25T12:04:00,175 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#A#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:00,175 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/9318826c66714c55b285c45a9ecd438c is 50, key is test_row_0/A:col10/1732536238786/Put/seqid=0 2024-11-25T12:04:00,178 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#B#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:00,179 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/78749d08131447e5a78afaed7ed02c88 is 50, key is test_row_0/B:col10/1732536238786/Put/seqid=0 2024-11-25T12:04:00,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741924_1100 (size=13289) 2024-11-25T12:04:00,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741925_1101 (size=13289) 2024-11-25T12:04:00,246 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/78749d08131447e5a78afaed7ed02c88 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/78749d08131447e5a78afaed7ed02c88 2024-11-25T12:04:00,257 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/B of 057d2058e8441af178fff3bc63c12b69 into 78749d08131447e5a78afaed7ed02c88(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:00,257 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:04:00,257 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/B, priority=13, startTime=1732536240139; duration=0sec 2024-11-25T12:04:00,258 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:00,258 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:B 2024-11-25T12:04:00,258 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:00,260 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:00,260 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 057d2058e8441af178fff3bc63c12b69/C is initiating minor compaction (all files) 2024-11-25T12:04:00,260 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 057d2058e8441af178fff3bc63c12b69/C in TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:00,261 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/ad476b63d9664fe2a18a327ce389fa48, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/3600a4686e894df9ad33c5d3a1ec4c0d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7627df3b7cf74e118ae63a66e42ab8db] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp, totalSize=36.9 K 2024-11-25T12:04:00,261 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting ad476b63d9664fe2a18a327ce389fa48, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732536236867 2024-11-25T12:04:00,262 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 3600a4686e894df9ad33c5d3a1ec4c0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732536237549 2024-11-25T12:04:00,263 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 7627df3b7cf74e118ae63a66e42ab8db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732536238786 2024-11-25T12:04:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-25T12:04:00,267 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:00,268 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:04:00,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-25T12:04:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T12:04:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T12:04:00,475 INFO [AsyncFSWAL-0-hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d-prefix:2bf237fd86ca,37643,1732536214674 {}] wal.AbstractFSWAL(1368): Slow sync cost: 176 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36647,DS-e355a87c-d052-4ab8-9e2a-3a6de0c1487f,DISK]] 2024-11-25T12:04:00,475 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:04:00,475 INFO [AsyncFSWAL-0-hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d-prefix:2bf237fd86ca,37643,1732536214674 {}] wal.AbstractFSWAL(1368): Slow sync cost: 175 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36647,DS-e355a87c-d052-4ab8-9e2a-3a6de0c1487f,DISK]] 2024-11-25T12:04:00,475 INFO [AsyncFSWAL-0-hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d-prefix:2bf237fd86ca,37643,1732536214674 {}] wal.AbstractFSWAL(1368): Slow sync cost: 157 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36647,DS-e355a87c-d052-4ab8-9e2a-3a6de0c1487f,DISK]] 2024-11-25T12:04:00,475 INFO [AsyncFSWAL-0-hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData-prefix:2bf237fd86ca,40019,1732536213609 {}] wal.AbstractFSWAL(1368): Slow sync cost: 199 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36647,DS-e355a87c-d052-4ab8-9e2a-3a6de0c1487f,DISK]] 2024-11-25T12:04:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:04:00,475 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:00,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:04:00,476 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-25T12:04:00,477 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:04:00,478 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 057d2058e8441af178fff3bc63c12b69#C#compaction#89 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:00,479 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/a0fc031d158b4b8089933cf4a492eee6 is 50, key is test_row_0/C:col10/1732536238786/Put/seqid=0 2024-11-25T12:04:00,481 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:04:00,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:04:00,484 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-25T12:04:00,484 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:04:00,485 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-25T12:04:00,485 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:04:00,486 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-25T12:04:00,486 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:04:00,487 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:00,487 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:00,488 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:00,488 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:00,488 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:00,488 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:04:00,488 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:04:00,489 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:00,489 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:04:00,490 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-25T12:04:00,490 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:04:00,490 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:00,491 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:00,491 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:04:00,491 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:00,492 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:00,492 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:04:00,492 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-25T12:04:00,492 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:04:00,492 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:00,492 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:00,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:04:00,497 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-25T12:04:00,497 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:04:00,497 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:00,497 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:00,497 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:04:00,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/69e32558683c48e3b5e6cc77b7be2b16 is 50, key is test_row_0/A:col10/1732536240317/Put/seqid=0 2024-11-25T12:04:00,500 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-25T12:04:00,500 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:04:00,500 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:00,501 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:00,501 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:04:00,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741926_1102 (size=13289) 2024-11-25T12:04:00,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741927_1103 (size=17181) 2024-11-25T12:04:00,514 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/a0fc031d158b4b8089933cf4a492eee6 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/a0fc031d158b4b8089933cf4a492eee6 2024-11-25T12:04:00,521 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/C of 057d2058e8441af178fff3bc63c12b69 into a0fc031d158b4b8089933cf4a492eee6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:00,521 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:04:00,521 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/C, priority=13, startTime=1732536240142; duration=0sec 2024-11-25T12:04:00,521 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:00,521 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:C 2024-11-25T12:04:00,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T12:04:00,622 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/9318826c66714c55b285c45a9ecd438c as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/9318826c66714c55b285c45a9ecd438c 2024-11-25T12:04:00,630 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 057d2058e8441af178fff3bc63c12b69/A of 057d2058e8441af178fff3bc63c12b69 into 9318826c66714c55b285c45a9ecd438c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:00,630 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:04:00,630 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69., storeName=057d2058e8441af178fff3bc63c12b69/A, priority=13, startTime=1732536240139; duration=0sec 2024-11-25T12:04:00,630 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:00,630 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 057d2058e8441af178fff3bc63c12b69:A 2024-11-25T12:04:00,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T12:04:00,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:00,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:04:00,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:00,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:00,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:00,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T12:04:00,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:00,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:04:00,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:00,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:00,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T12:04:00,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/69e32558683c48e3b5e6cc77b7be2b16 2024-11-25T12:04:00,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/54f72a38d036498da4492b4882b2cfe3 is 50, key is test_row_0/B:col10/1732536240317/Put/seqid=0 2024-11-25T12:04:00,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741928_1104 (size=12301) 2024-11-25T12:04:00,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T12:04:00,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:00,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:04:00,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:00,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:00,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:01,003 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-25T12:04:01,003 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:04:01,003 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:01,003 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:01,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:04:01,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T12:04:01,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:01,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:04:01,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:01,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:01,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:01,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T12:04:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:04:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:01,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:01,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/54f72a38d036498da4492b4882b2cfe3 2024-11-25T12:04:01,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/535d24fd588941f4b741a283518e6cef is 50, key is test_row_0/C:col10/1732536240317/Put/seqid=0 2024-11-25T12:04:01,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741929_1105 (size=12301) 2024-11-25T12:04:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T12:04:01,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T12:04:01,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:01,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:04:01,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:01,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:01,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:01,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T12:04:01,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:01,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:04:01,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:01,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:01,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:01,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T12:04:01,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:01,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. as already flushing 2024-11-25T12:04:01,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:01,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:01,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:01,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:01,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/535d24fd588941f4b741a283518e6cef 2024-11-25T12:04:01,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/69e32558683c48e3b5e6cc77b7be2b16 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/69e32558683c48e3b5e6cc77b7be2b16 2024-11-25T12:04:01,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/69e32558683c48e3b5e6cc77b7be2b16, entries=250, sequenceid=444, filesize=16.8 K 2024-11-25T12:04:01,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/54f72a38d036498da4492b4882b2cfe3 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/54f72a38d036498da4492b4882b2cfe3 2024-11-25T12:04:01,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/54f72a38d036498da4492b4882b2cfe3, entries=150, sequenceid=444, filesize=12.0 K 2024-11-25T12:04:01,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/535d24fd588941f4b741a283518e6cef as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/535d24fd588941f4b741a283518e6cef 2024-11-25T12:04:01,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/535d24fd588941f4b741a283518e6cef, entries=150, sequenceid=444, filesize=12.0 K 2024-11-25T12:04:01,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=13.42 KB/13740 for 057d2058e8441af178fff3bc63c12b69 in 1358ms, sequenceid=444, compaction requested=false 2024-11-25T12:04:01,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:04:01,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-25T12:04:01,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:01,897 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 057d2058e8441af178fff3bc63c12b69 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-25T12:04:01,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/a7e105100cd84c5eabd76c158f60f962 is 50, key is test_row_2/A:col10/1732536240993/Put/seqid=0 2024-11-25T12:04:01,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741930_1106 (size=7415) 2024-11-25T12:04:01,926 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/a7e105100cd84c5eabd76c158f60f962 2024-11-25T12:04:01,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/c55c166f3e214433bf63ca0e66deada6 is 50, key is test_row_2/B:col10/1732536240993/Put/seqid=0 2024-11-25T12:04:01,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741931_1107 (size=7415) 2024-11-25T12:04:02,374 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/c55c166f3e214433bf63ca0e66deada6 2024-11-25T12:04:02,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/5b6dc8085349476c91ef9d1f8813c2a9 is 50, key is test_row_2/C:col10/1732536240993/Put/seqid=0 2024-11-25T12:04:02,413 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T12:04:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T12:04:02,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741932_1108 (size=7415) 2024-11-25T12:04:02,439 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/5b6dc8085349476c91ef9d1f8813c2a9 2024-11-25T12:04:02,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/A/a7e105100cd84c5eabd76c158f60f962 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/a7e105100cd84c5eabd76c158f60f962 2024-11-25T12:04:02,464 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/a7e105100cd84c5eabd76c158f60f962, entries=50, sequenceid=451, filesize=7.2 K 2024-11-25T12:04:02,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/B/c55c166f3e214433bf63ca0e66deada6 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/c55c166f3e214433bf63ca0e66deada6 2024-11-25T12:04:02,488 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/c55c166f3e214433bf63ca0e66deada6, entries=50, sequenceid=451, filesize=7.2 K 2024-11-25T12:04:02,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/.tmp/C/5b6dc8085349476c91ef9d1f8813c2a9 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/5b6dc8085349476c91ef9d1f8813c2a9 2024-11-25T12:04:02,543 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/5b6dc8085349476c91ef9d1f8813c2a9, entries=50, sequenceid=451, filesize=7.2 K 2024-11-25T12:04:02,544 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 057d2058e8441af178fff3bc63c12b69 in 647ms, sequenceid=451, compaction requested=true 2024-11-25T12:04:02,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 057d2058e8441af178fff3bc63c12b69: 2024-11-25T12:04:02,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:02,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-25T12:04:02,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-25T12:04:02,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-25T12:04:02,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0680 sec 2024-11-25T12:04:02,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.2840 sec 2024-11-25T12:04:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-25T12:04:04,427 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:04,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T12:04:04,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-25T12:04:04,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-25T12:04:04,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-25T12:04:04,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-25T12:04:04,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-25T12:04:04,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T12:04:04,427 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1835 2024-11-25T12:04:04,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1768 2024-11-25T12:04:04,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T12:04:04,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 892 2024-11-25T12:04:04,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2676 rows 2024-11-25T12:04:04,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 882 2024-11-25T12:04:04,428 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2632 rows 2024-11-25T12:04:04,428 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T12:04:04,428 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:04:04,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:04,428 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:04,431 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:04:04,433 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-25T12:04:04,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T12:04:04,443 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536244442"}]},"ts":"1732536244442"} 2024-11-25T12:04:04,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T12:04:04,445 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T12:04:04,445 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T12:04:04,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T12:04:04,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=057d2058e8441af178fff3bc63c12b69, UNASSIGN}] 2024-11-25T12:04:04,453 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=057d2058e8441af178fff3bc63c12b69, UNASSIGN 2024-11-25T12:04:04,455 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=057d2058e8441af178fff3bc63c12b69, regionState=CLOSING, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:04,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=057d2058e8441af178fff3bc63c12b69, UNASSIGN because future has completed 2024-11-25T12:04:04,457 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T12:04:04,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674}] 2024-11-25T12:04:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T12:04:04,621 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:04:04,621 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T12:04:04,622 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 057d2058e8441af178fff3bc63c12b69, disabling compactions & flushes 2024-11-25T12:04:04,622 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:04,622 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:04,622 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. after waiting 0 ms 2024-11-25T12:04:04,622 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:04,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/be598eaa5304408aa2ea53463fb8dfcf, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/8113cc61c64945baa16ec7baa26d62e3, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/e4f58bc6c0b747a78ed6c454dd7edc36, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/244ba09f29f242678c5bca4f96f072d7, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/c7215971628d4c3fb112c59dbcdb0133, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/3b6f93352e8f43aab4edb06c6cd2e5ac, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/d5656e5dfd14433898e04907b74420f2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/a13bc658c7f34c0d809a54f6de511254, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/e0be88cbf187412eb2bd7b4e1c7d862e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/50b2e40a15cb471fb9e619f81e434a24, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/218536f9204b46d49460a35b01666e9d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/caa9234026f24447a7b4b95afa20c9f9, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/b561ff7d029a4012b60ffade52ca4bf0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/5cb5a1b237fc4016a3e002cd97635661, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/4d7b2c6634bb4677bdb2f5f2b52e7825, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/a7812e48946249f684011f396578aa5d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/b270257e32624e72a392fcbb69c93ba8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/04f3f39b9fe64717bdefa742d7c1e103, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/e0d04186093642e09b3ca352b5b1db6b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/0cbbe4ca41f24c8697a1c6f3969f2b7a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/c2a91bd2a0e54a77abd21f71033030da, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/a2412afb703e4b7ba90a28dca55f7910, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/5efc0c396a194a95a80d19a2a0ae161e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/31363a02e59f4d73a8eada9bca2ab247, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/ccff0d972cbf41bab54b41dde0fcacb3, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/0546aa67b7344e99b6009e1dde88b07a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/3de878cba7304cd7af91c69ae0332600, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/9ea15456c4c542e99fed36eb66cc9516, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/b6e99b8a4e9348b28cb6a3efaa664a10] to archive 2024-11-25T12:04:04,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:04:04,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/be598eaa5304408aa2ea53463fb8dfcf to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/be598eaa5304408aa2ea53463fb8dfcf 2024-11-25T12:04:04,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/8113cc61c64945baa16ec7baa26d62e3 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/8113cc61c64945baa16ec7baa26d62e3 2024-11-25T12:04:04,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/e4f58bc6c0b747a78ed6c454dd7edc36 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/e4f58bc6c0b747a78ed6c454dd7edc36 2024-11-25T12:04:04,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/244ba09f29f242678c5bca4f96f072d7 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/244ba09f29f242678c5bca4f96f072d7 2024-11-25T12:04:04,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/c7215971628d4c3fb112c59dbcdb0133 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/c7215971628d4c3fb112c59dbcdb0133 2024-11-25T12:04:04,656 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/3b6f93352e8f43aab4edb06c6cd2e5ac to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/3b6f93352e8f43aab4edb06c6cd2e5ac 2024-11-25T12:04:04,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/d5656e5dfd14433898e04907b74420f2 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/d5656e5dfd14433898e04907b74420f2 2024-11-25T12:04:04,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/a13bc658c7f34c0d809a54f6de511254 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/a13bc658c7f34c0d809a54f6de511254 2024-11-25T12:04:04,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/e0be88cbf187412eb2bd7b4e1c7d862e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/e0be88cbf187412eb2bd7b4e1c7d862e 2024-11-25T12:04:04,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/50b2e40a15cb471fb9e619f81e434a24 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/50b2e40a15cb471fb9e619f81e434a24 2024-11-25T12:04:04,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/218536f9204b46d49460a35b01666e9d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/218536f9204b46d49460a35b01666e9d 2024-11-25T12:04:04,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/caa9234026f24447a7b4b95afa20c9f9 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/caa9234026f24447a7b4b95afa20c9f9 2024-11-25T12:04:04,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/b561ff7d029a4012b60ffade52ca4bf0 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/b561ff7d029a4012b60ffade52ca4bf0 2024-11-25T12:04:04,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/5cb5a1b237fc4016a3e002cd97635661 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/5cb5a1b237fc4016a3e002cd97635661 2024-11-25T12:04:04,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/4d7b2c6634bb4677bdb2f5f2b52e7825 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/4d7b2c6634bb4677bdb2f5f2b52e7825 2024-11-25T12:04:04,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/a7812e48946249f684011f396578aa5d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/a7812e48946249f684011f396578aa5d 2024-11-25T12:04:04,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/b270257e32624e72a392fcbb69c93ba8 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/b270257e32624e72a392fcbb69c93ba8 2024-11-25T12:04:04,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/04f3f39b9fe64717bdefa742d7c1e103 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/04f3f39b9fe64717bdefa742d7c1e103 2024-11-25T12:04:04,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/e0d04186093642e09b3ca352b5b1db6b to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/e0d04186093642e09b3ca352b5b1db6b 2024-11-25T12:04:04,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/0cbbe4ca41f24c8697a1c6f3969f2b7a to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/0cbbe4ca41f24c8697a1c6f3969f2b7a 2024-11-25T12:04:04,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/c2a91bd2a0e54a77abd21f71033030da to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/c2a91bd2a0e54a77abd21f71033030da 2024-11-25T12:04:04,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/a2412afb703e4b7ba90a28dca55f7910 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/a2412afb703e4b7ba90a28dca55f7910 2024-11-25T12:04:04,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/5efc0c396a194a95a80d19a2a0ae161e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/5efc0c396a194a95a80d19a2a0ae161e 2024-11-25T12:04:04,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/31363a02e59f4d73a8eada9bca2ab247 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/31363a02e59f4d73a8eada9bca2ab247 2024-11-25T12:04:04,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/ccff0d972cbf41bab54b41dde0fcacb3 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/ccff0d972cbf41bab54b41dde0fcacb3 2024-11-25T12:04:04,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/0546aa67b7344e99b6009e1dde88b07a to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/0546aa67b7344e99b6009e1dde88b07a 2024-11-25T12:04:04,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/3de878cba7304cd7af91c69ae0332600 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/3de878cba7304cd7af91c69ae0332600 2024-11-25T12:04:04,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/9ea15456c4c542e99fed36eb66cc9516 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/9ea15456c4c542e99fed36eb66cc9516 2024-11-25T12:04:04,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/b6e99b8a4e9348b28cb6a3efaa664a10 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/b6e99b8a4e9348b28cb6a3efaa664a10 2024-11-25T12:04:04,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/68f75de0725e46bd8ddb6b44292db658, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/c17a8d757e6244dab1615cd644265c42, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/518ea8c508c145a2989449e309093be4, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/74fd1c6ebc8f4d53b2db05a05b6018a2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/2dffcd7f8b444eb08434c21d1a4755ee, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7e5232390e3245269f5f4d2ce59aa807, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/bfa1699ec4c745599838c8adb0d111e1, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/e9d6c6e15c304d86a14ccb3cabd3e0a5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/bb74317b3d31480b817b048daff7ba41, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/881068bb4d4a47a2af7ac45ae72f0c54, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/bb0f252910ab42bcac0f275328c166f8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/f52905948f4e44a9b3db268b69ad0174, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7bb1e4719f3b4fa9b9122ee81b97adc8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/b7df0f6080024fb68324baefdc714dc5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7ba62eaebbf04edd83a98f7f911382f2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/898a22bfd0854385838a19f0f68b389c, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/719cae236cc948a99c4ae4eff05d78be, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/e81881fd6c93489ababa5e8994366405, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/301e5398a9b84863a6e7fc9b37c8f98f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/d8c968509e804ae89f8e7773ed6552b8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/8a361dabd41347f7a51beaea3a881126, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/41d83c31506e41b4b593d0fbfba36d17, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/027c3970774c4453af21c253e8111fa4, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/180f82cce61246c9acf76dc068f96778, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7ff7ca3ccc00422bacd0f6f8317044c4, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/10fa1b5a122446038e89209e76fc4453, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/116b6919129f4a0a98df6fe2a8741bd0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/f36067ea3b5f4c1c8b0709aa96853c20, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/9c6244dd4cc643a1b12fbcdd09547ee5] to archive 2024-11-25T12:04:04,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:04:04,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/68f75de0725e46bd8ddb6b44292db658 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/68f75de0725e46bd8ddb6b44292db658 2024-11-25T12:04:04,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/c17a8d757e6244dab1615cd644265c42 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/c17a8d757e6244dab1615cd644265c42 2024-11-25T12:04:04,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/518ea8c508c145a2989449e309093be4 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/518ea8c508c145a2989449e309093be4 2024-11-25T12:04:04,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/74fd1c6ebc8f4d53b2db05a05b6018a2 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/74fd1c6ebc8f4d53b2db05a05b6018a2 2024-11-25T12:04:04,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/2dffcd7f8b444eb08434c21d1a4755ee to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/2dffcd7f8b444eb08434c21d1a4755ee 2024-11-25T12:04:04,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7e5232390e3245269f5f4d2ce59aa807 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7e5232390e3245269f5f4d2ce59aa807 2024-11-25T12:04:04,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/bfa1699ec4c745599838c8adb0d111e1 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/bfa1699ec4c745599838c8adb0d111e1 2024-11-25T12:04:04,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/e9d6c6e15c304d86a14ccb3cabd3e0a5 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/e9d6c6e15c304d86a14ccb3cabd3e0a5 2024-11-25T12:04:04,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/bb74317b3d31480b817b048daff7ba41 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/bb74317b3d31480b817b048daff7ba41 2024-11-25T12:04:04,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/881068bb4d4a47a2af7ac45ae72f0c54 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/881068bb4d4a47a2af7ac45ae72f0c54 2024-11-25T12:04:04,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/bb0f252910ab42bcac0f275328c166f8 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/bb0f252910ab42bcac0f275328c166f8 2024-11-25T12:04:04,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/f52905948f4e44a9b3db268b69ad0174 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/f52905948f4e44a9b3db268b69ad0174 2024-11-25T12:04:04,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7bb1e4719f3b4fa9b9122ee81b97adc8 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7bb1e4719f3b4fa9b9122ee81b97adc8 2024-11-25T12:04:04,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/b7df0f6080024fb68324baefdc714dc5 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/b7df0f6080024fb68324baefdc714dc5 2024-11-25T12:04:04,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7ba62eaebbf04edd83a98f7f911382f2 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7ba62eaebbf04edd83a98f7f911382f2 2024-11-25T12:04:04,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/898a22bfd0854385838a19f0f68b389c to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/898a22bfd0854385838a19f0f68b389c 2024-11-25T12:04:04,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/719cae236cc948a99c4ae4eff05d78be to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/719cae236cc948a99c4ae4eff05d78be 2024-11-25T12:04:04,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/e81881fd6c93489ababa5e8994366405 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/e81881fd6c93489ababa5e8994366405 2024-11-25T12:04:04,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/301e5398a9b84863a6e7fc9b37c8f98f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/301e5398a9b84863a6e7fc9b37c8f98f 2024-11-25T12:04:04,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T12:04:04,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/d8c968509e804ae89f8e7773ed6552b8 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/d8c968509e804ae89f8e7773ed6552b8 2024-11-25T12:04:04,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/8a361dabd41347f7a51beaea3a881126 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/8a361dabd41347f7a51beaea3a881126 2024-11-25T12:04:04,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/41d83c31506e41b4b593d0fbfba36d17 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/41d83c31506e41b4b593d0fbfba36d17 2024-11-25T12:04:04,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/027c3970774c4453af21c253e8111fa4 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/027c3970774c4453af21c253e8111fa4 2024-11-25T12:04:04,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/180f82cce61246c9acf76dc068f96778 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/180f82cce61246c9acf76dc068f96778 2024-11-25T12:04:04,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7ff7ca3ccc00422bacd0f6f8317044c4 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/7ff7ca3ccc00422bacd0f6f8317044c4 2024-11-25T12:04:04,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/10fa1b5a122446038e89209e76fc4453 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/10fa1b5a122446038e89209e76fc4453 2024-11-25T12:04:04,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/116b6919129f4a0a98df6fe2a8741bd0 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/116b6919129f4a0a98df6fe2a8741bd0 2024-11-25T12:04:04,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/f36067ea3b5f4c1c8b0709aa96853c20 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/f36067ea3b5f4c1c8b0709aa96853c20 2024-11-25T12:04:04,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/9c6244dd4cc643a1b12fbcdd09547ee5 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/9c6244dd4cc643a1b12fbcdd09547ee5 2024-11-25T12:04:04,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/e196cced833b4c5e93a16cfb31eeba7b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/c84bfb37fca44df595a3323345869c80, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/63502da6c822477a80e776f337720cd8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/2c8bcf09bf8b4c7c8a555ab747a72db3, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/2141ed26e5d14f118e095020fa141fc6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/55a08b11b17c4113a3fa3d777a0296f9, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/16da9fbbf0aa4ee3a1596654a2aa2c7d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/575bf187480d4ee5861512eca7664197, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/5389a83e64484b41afe9f0c3520c16fc, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/5f1230b505c2475fa079c5b9b986292f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/54d99e1e87af471f8cb1f08328f45967, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/8a4fe0509c554d49af145ac7d333b908, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/dce8846486fa47d78d89499186fa189b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/fe37d773be3a4d94bc66dbbc5562b056, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/1cd4aab66c7d4a368a708eee4e1dd847, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/63f466bb74d644efba621b7b3272896c, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/c2ee3165ad6b45fc9e1f2ca0c7214b6d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7aa9400864a345c086bf0c6ba35202cd, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/2f624b0b614f425986826477f5942e51, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/72ad8166874e4e8cacf860ae0c4b8a95, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/194d532317b94621abb3a08ebddd650e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/625f140072784fa9994dd8ec6ee14b62, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7326fb52e8a54faf8548ce61c9da828a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/b4c416c687a74efb82113030b52fd50e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7b8d4ee868b449ffb5402b6dd96083a6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/ad476b63d9664fe2a18a327ce389fa48, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/c61bb1ad28c44268bc7089a5b4b7e9e2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/3600a4686e894df9ad33c5d3a1ec4c0d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7627df3b7cf74e118ae63a66e42ab8db] to archive 2024-11-25T12:04:04,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:04:04,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/e196cced833b4c5e93a16cfb31eeba7b to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/e196cced833b4c5e93a16cfb31eeba7b 2024-11-25T12:04:04,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/c84bfb37fca44df595a3323345869c80 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/c84bfb37fca44df595a3323345869c80 2024-11-25T12:04:04,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/63502da6c822477a80e776f337720cd8 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/63502da6c822477a80e776f337720cd8 2024-11-25T12:04:04,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/2c8bcf09bf8b4c7c8a555ab747a72db3 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/2c8bcf09bf8b4c7c8a555ab747a72db3 2024-11-25T12:04:04,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/2141ed26e5d14f118e095020fa141fc6 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/2141ed26e5d14f118e095020fa141fc6 2024-11-25T12:04:04,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/55a08b11b17c4113a3fa3d777a0296f9 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/55a08b11b17c4113a3fa3d777a0296f9 2024-11-25T12:04:04,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/16da9fbbf0aa4ee3a1596654a2aa2c7d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/16da9fbbf0aa4ee3a1596654a2aa2c7d 2024-11-25T12:04:04,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/575bf187480d4ee5861512eca7664197 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/575bf187480d4ee5861512eca7664197 2024-11-25T12:04:04,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/5389a83e64484b41afe9f0c3520c16fc to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/5389a83e64484b41afe9f0c3520c16fc 2024-11-25T12:04:04,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/5f1230b505c2475fa079c5b9b986292f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/5f1230b505c2475fa079c5b9b986292f 2024-11-25T12:04:04,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/54d99e1e87af471f8cb1f08328f45967 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/54d99e1e87af471f8cb1f08328f45967 2024-11-25T12:04:04,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/8a4fe0509c554d49af145ac7d333b908 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/8a4fe0509c554d49af145ac7d333b908 2024-11-25T12:04:04,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/dce8846486fa47d78d89499186fa189b to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/dce8846486fa47d78d89499186fa189b 2024-11-25T12:04:04,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/fe37d773be3a4d94bc66dbbc5562b056 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/fe37d773be3a4d94bc66dbbc5562b056 2024-11-25T12:04:04,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/1cd4aab66c7d4a368a708eee4e1dd847 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/1cd4aab66c7d4a368a708eee4e1dd847 2024-11-25T12:04:04,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/63f466bb74d644efba621b7b3272896c to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/63f466bb74d644efba621b7b3272896c 2024-11-25T12:04:04,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/c2ee3165ad6b45fc9e1f2ca0c7214b6d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/c2ee3165ad6b45fc9e1f2ca0c7214b6d 2024-11-25T12:04:04,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7aa9400864a345c086bf0c6ba35202cd to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7aa9400864a345c086bf0c6ba35202cd 2024-11-25T12:04:04,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/2f624b0b614f425986826477f5942e51 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/2f624b0b614f425986826477f5942e51 2024-11-25T12:04:04,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/72ad8166874e4e8cacf860ae0c4b8a95 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/72ad8166874e4e8cacf860ae0c4b8a95 2024-11-25T12:04:04,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/194d532317b94621abb3a08ebddd650e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/194d532317b94621abb3a08ebddd650e 2024-11-25T12:04:04,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/625f140072784fa9994dd8ec6ee14b62 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/625f140072784fa9994dd8ec6ee14b62 2024-11-25T12:04:04,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7326fb52e8a54faf8548ce61c9da828a to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7326fb52e8a54faf8548ce61c9da828a 2024-11-25T12:04:04,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/b4c416c687a74efb82113030b52fd50e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/b4c416c687a74efb82113030b52fd50e 2024-11-25T12:04:04,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7b8d4ee868b449ffb5402b6dd96083a6 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7b8d4ee868b449ffb5402b6dd96083a6 2024-11-25T12:04:04,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/ad476b63d9664fe2a18a327ce389fa48 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/ad476b63d9664fe2a18a327ce389fa48 2024-11-25T12:04:04,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/c61bb1ad28c44268bc7089a5b4b7e9e2 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/c61bb1ad28c44268bc7089a5b4b7e9e2 2024-11-25T12:04:04,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/3600a4686e894df9ad33c5d3a1ec4c0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/3600a4686e894df9ad33c5d3a1ec4c0d 2024-11-25T12:04:04,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7627df3b7cf74e118ae63a66e42ab8db to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/7627df3b7cf74e118ae63a66e42ab8db 2024-11-25T12:04:04,869 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/recovered.edits/454.seqid, newMaxSeqId=454, maxSeqId=1 2024-11-25T12:04:04,870 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69. 2024-11-25T12:04:04,870 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 057d2058e8441af178fff3bc63c12b69: Waiting for close lock at 1732536244622Running coprocessor pre-close hooks at 1732536244622Disabling compacts and flushes for region at 1732536244622Disabling writes for close at 1732536244622Writing region close event to WAL at 1732536244864 (+242 ms)Running coprocessor post-close hooks at 1732536244870 (+6 ms)Closed at 1732536244870 2024-11-25T12:04:04,873 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 057d2058e8441af178fff3bc63c12b69 2024-11-25T12:04:04,873 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=057d2058e8441af178fff3bc63c12b69, regionState=CLOSED 2024-11-25T12:04:04,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 because future has completed 2024-11-25T12:04:04,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-25T12:04:04,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 057d2058e8441af178fff3bc63c12b69, server=2bf237fd86ca,37643,1732536214674 in 420 msec 2024-11-25T12:04:04,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-25T12:04:04,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=057d2058e8441af178fff3bc63c12b69, UNASSIGN in 428 msec 2024-11-25T12:04:04,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-25T12:04:04,890 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 440 msec 2024-11-25T12:04:04,892 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536244891"}]},"ts":"1732536244891"} 2024-11-25T12:04:04,893 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T12:04:04,893 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T12:04:04,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 459 msec 2024-11-25T12:04:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T12:04:05,067 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:05,071 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-25T12:04:05,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:04:05,079 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:04:05,081 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:04:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T12:04:05,092 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69 2024-11-25T12:04:05,099 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/recovered.edits] 2024-11-25T12:04:05,104 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/69e32558683c48e3b5e6cc77b7be2b16 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/69e32558683c48e3b5e6cc77b7be2b16 2024-11-25T12:04:05,106 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/9318826c66714c55b285c45a9ecd438c to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/9318826c66714c55b285c45a9ecd438c 2024-11-25T12:04:05,116 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/a7e105100cd84c5eabd76c158f60f962 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/A/a7e105100cd84c5eabd76c158f60f962 2024-11-25T12:04:05,120 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/54f72a38d036498da4492b4882b2cfe3 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/54f72a38d036498da4492b4882b2cfe3 2024-11-25T12:04:05,122 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/78749d08131447e5a78afaed7ed02c88 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/78749d08131447e5a78afaed7ed02c88 2024-11-25T12:04:05,124 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/c55c166f3e214433bf63ca0e66deada6 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/B/c55c166f3e214433bf63ca0e66deada6 2024-11-25T12:04:05,127 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/535d24fd588941f4b741a283518e6cef to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/535d24fd588941f4b741a283518e6cef 2024-11-25T12:04:05,130 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/5b6dc8085349476c91ef9d1f8813c2a9 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/5b6dc8085349476c91ef9d1f8813c2a9 2024-11-25T12:04:05,137 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/a0fc031d158b4b8089933cf4a492eee6 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/C/a0fc031d158b4b8089933cf4a492eee6 2024-11-25T12:04:05,162 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/recovered.edits/454.seqid to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69/recovered.edits/454.seqid 2024-11-25T12:04:05,164 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/057d2058e8441af178fff3bc63c12b69 2024-11-25T12:04:05,165 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T12:04:05,177 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:04:05,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-25T12:04:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T12:04:05,191 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T12:04:05,242 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T12:04:05,244 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:04:05,244 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T12:04:05,245 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732536245244"}]},"ts":"9223372036854775807"} 2024-11-25T12:04:05,251 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T12:04:05,251 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 057d2058e8441af178fff3bc63c12b69, NAME => 'TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T12:04:05,252 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T12:04:05,252 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732536245252"}]},"ts":"9223372036854775807"} 2024-11-25T12:04:05,255 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T12:04:05,257 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:04:05,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 185 msec 2024-11-25T12:04:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-25T12:04:05,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T12:04:05,398 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:05,415 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 210) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2009888188_22 at /127.0.0.1:48752 [Waiting for operation #127] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;2bf237fd86ca:37643-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2009888188_22 at /127.0.0.1:38358 [Waiting for operation #241] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-514927083_22 at /127.0.0.1:38494 [Waiting for operation #233] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1076 (was 900) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4792 (was 5827) 2024-11-25T12:04:05,431 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=1076, ProcessCount=11, AvailableMemoryMB=4791 2024-11-25T12:04:05,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T12:04:05,433 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:04:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T12:04:05,436 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T12:04:05,436 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:05,436 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-25T12:04:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-25T12:04:05,438 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T12:04:05,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741933_1109 (size=959) 2024-11-25T12:04:05,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-25T12:04:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-25T12:04:05,867 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d 2024-11-25T12:04:05,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741934_1110 (size=53) 2024-11-25T12:04:05,875 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:04:05,876 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 14ed09f1e402e2ed659ccb58744c29bb, disabling compactions & flushes 2024-11-25T12:04:05,876 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:05,876 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:05,876 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. after waiting 0 ms 2024-11-25T12:04:05,876 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:05,876 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:05,876 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 14ed09f1e402e2ed659ccb58744c29bb: Waiting for close lock at 1732536245876Disabling compacts and flushes for region at 1732536245876Disabling writes for close at 1732536245876Writing region close event to WAL at 1732536245876Closed at 1732536245876 2024-11-25T12:04:05,880 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T12:04:05,880 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732536245880"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732536245880"}]},"ts":"1732536245880"} 2024-11-25T12:04:05,883 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T12:04:05,884 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T12:04:05,884 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536245884"}]},"ts":"1732536245884"} 2024-11-25T12:04:05,886 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T12:04:05,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ed09f1e402e2ed659ccb58744c29bb, ASSIGN}] 2024-11-25T12:04:05,888 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ed09f1e402e2ed659ccb58744c29bb, ASSIGN 2024-11-25T12:04:05,889 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ed09f1e402e2ed659ccb58744c29bb, ASSIGN; state=OFFLINE, location=2bf237fd86ca,37643,1732536214674; forceNewPlan=false, retain=false 2024-11-25T12:04:06,040 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=14ed09f1e402e2ed659ccb58744c29bb, regionState=OPENING, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:06,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ed09f1e402e2ed659ccb58744c29bb, ASSIGN because future has completed 2024-11-25T12:04:06,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674}] 2024-11-25T12:04:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-25T12:04:06,210 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:06,211 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:04:06,211 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:06,212 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:04:06,212 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:06,212 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:06,214 INFO [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:06,217 INFO [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14ed09f1e402e2ed659ccb58744c29bb columnFamilyName A 2024-11-25T12:04:06,217 DEBUG [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:06,218 INFO [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] regionserver.HStore(327): Store=14ed09f1e402e2ed659ccb58744c29bb/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:04:06,218 INFO [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:06,220 INFO [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14ed09f1e402e2ed659ccb58744c29bb columnFamilyName B 2024-11-25T12:04:06,220 DEBUG [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:06,221 INFO [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] regionserver.HStore(327): Store=14ed09f1e402e2ed659ccb58744c29bb/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:04:06,221 INFO [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:06,224 INFO [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14ed09f1e402e2ed659ccb58744c29bb columnFamilyName C 2024-11-25T12:04:06,224 DEBUG [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:06,224 INFO [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] regionserver.HStore(327): Store=14ed09f1e402e2ed659ccb58744c29bb/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:04:06,225 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:06,225 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:06,226 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:06,227 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:06,227 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:06,228 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:04:06,232 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:06,236 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:04:06,237 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 14ed09f1e402e2ed659ccb58744c29bb; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72144967, jitterRate=0.07504378259181976}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:04:06,237 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:06,238 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 14ed09f1e402e2ed659ccb58744c29bb: Running coprocessor pre-open hook at 1732536246212Writing region info on filesystem at 1732536246212Initializing all the Stores at 1732536246213 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536246213Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536246213Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536246214 (+1 ms)Cleaning up temporary data from old regions at 1732536246227 (+13 ms)Running coprocessor post-open hooks at 1732536246237 (+10 ms)Region opened successfully at 1732536246238 (+1 ms) 2024-11-25T12:04:06,239 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., pid=32, masterSystemTime=1732536246205 2024-11-25T12:04:06,242 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=14ed09f1e402e2ed659ccb58744c29bb, regionState=OPEN, openSeqNum=2, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:06,245 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:06,245 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:06,245 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 because future has completed 2024-11-25T12:04:06,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-25T12:04:06,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 in 213 msec 2024-11-25T12:04:06,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-25T12:04:06,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ed09f1e402e2ed659ccb58744c29bb, ASSIGN in 373 msec 2024-11-25T12:04:06,264 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T12:04:06,264 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536246264"}]},"ts":"1732536246264"} 2024-11-25T12:04:06,266 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T12:04:06,270 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T12:04:06,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 837 msec 2024-11-25T12:04:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-25T12:04:06,578 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:06,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d44c0b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:06,579 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:06,579 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:06,581 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:06,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:06,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:06,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7640c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:06,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:06,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:06,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:06,584 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:06,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@329c9ead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:06,586 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:06,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:06,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:06,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:06,591 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T12:04:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 2bf237fd86ca,40019,1732536213609 2024-11-25T12:04:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1eb25f6a 2024-11-25T12:04:06,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T12:04:06,593 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34158, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T12:04:06,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T12:04:06,600 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:04:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-25T12:04:06,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741935_1111 (size=995) 2024-11-25T12:04:06,639 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-25T12:04:06,639 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-25T12:04:06,644 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T12:04:06,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ed09f1e402e2ed659ccb58744c29bb, REOPEN/MOVE}] 2024-11-25T12:04:06,653 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ed09f1e402e2ed659ccb58744c29bb, REOPEN/MOVE 2024-11-25T12:04:06,654 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=14ed09f1e402e2ed659ccb58744c29bb, regionState=CLOSING, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:06,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ed09f1e402e2ed659ccb58744c29bb, REOPEN/MOVE because future has completed 2024-11-25T12:04:06,656 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T12:04:06,656 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674}] 2024-11-25T12:04:06,838 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:06,839 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T12:04:06,839 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 14ed09f1e402e2ed659ccb58744c29bb, disabling compactions & flushes 2024-11-25T12:04:06,839 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:06,839 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:06,839 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. after waiting 0 ms 2024-11-25T12:04:06,839 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:06,845 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T12:04:06,846 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:06,852 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 14ed09f1e402e2ed659ccb58744c29bb: Waiting for close lock at 1732536246839Running coprocessor pre-close hooks at 1732536246839Disabling compacts and flushes for region at 1732536246839Disabling writes for close at 1732536246839Writing region close event to WAL at 1732536246839Running coprocessor post-close hooks at 1732536246846 (+7 ms)Closed at 1732536246846 2024-11-25T12:04:06,853 WARN [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 14ed09f1e402e2ed659ccb58744c29bb to self. 2024-11-25T12:04:06,857 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:06,857 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=14ed09f1e402e2ed659ccb58744c29bb, regionState=CLOSED 2024-11-25T12:04:06,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 because future has completed 2024-11-25T12:04:06,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-25T12:04:06,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 in 205 msec 2024-11-25T12:04:06,864 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ed09f1e402e2ed659ccb58744c29bb, REOPEN/MOVE; state=CLOSED, location=2bf237fd86ca,37643,1732536214674; forceNewPlan=false, retain=true 2024-11-25T12:04:07,015 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=14ed09f1e402e2ed659ccb58744c29bb, regionState=OPENING, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:07,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ed09f1e402e2ed659ccb58744c29bb, REOPEN/MOVE because future has completed 2024-11-25T12:04:07,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674}] 2024-11-25T12:04:07,176 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:07,176 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:04:07,176 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:07,177 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:04:07,177 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:07,177 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:07,187 INFO [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:07,195 INFO [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14ed09f1e402e2ed659ccb58744c29bb columnFamilyName A 2024-11-25T12:04:07,198 DEBUG [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:07,198 INFO [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] regionserver.HStore(327): Store=14ed09f1e402e2ed659ccb58744c29bb/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:04:07,199 INFO [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:07,200 INFO [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14ed09f1e402e2ed659ccb58744c29bb columnFamilyName B 2024-11-25T12:04:07,200 DEBUG [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:07,200 INFO [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] regionserver.HStore(327): Store=14ed09f1e402e2ed659ccb58744c29bb/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:04:07,200 INFO [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:07,201 INFO [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14ed09f1e402e2ed659ccb58744c29bb columnFamilyName C 2024-11-25T12:04:07,201 DEBUG [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:07,202 INFO [StoreOpener-14ed09f1e402e2ed659ccb58744c29bb-1 {}] regionserver.HStore(327): Store=14ed09f1e402e2ed659ccb58744c29bb/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:04:07,202 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:07,203 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:07,204 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:07,205 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:07,205 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:07,206 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:04:07,207 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:07,208 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 14ed09f1e402e2ed659ccb58744c29bb; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64760444, jitterRate=-0.03499418497085571}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:04:07,208 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:07,209 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 14ed09f1e402e2ed659ccb58744c29bb: Running coprocessor pre-open hook at 1732536247177Writing region info on filesystem at 1732536247177Initializing all the Stores at 1732536247182 (+5 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536247182Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536247182Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536247182Cleaning up temporary data from old regions at 1732536247205 (+23 ms)Running coprocessor post-open hooks at 1732536247208 (+3 ms)Region opened successfully at 1732536247209 (+1 ms) 2024-11-25T12:04:07,210 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., pid=37, masterSystemTime=1732536247171 2024-11-25T12:04:07,213 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:07,213 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:07,213 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=14ed09f1e402e2ed659ccb58744c29bb, regionState=OPEN, openSeqNum=5, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:07,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 because future has completed 2024-11-25T12:04:07,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-25T12:04:07,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 in 198 msec 2024-11-25T12:04:07,221 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-25T12:04:07,221 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ed09f1e402e2ed659ccb58744c29bb, REOPEN/MOVE in 566 msec 2024-11-25T12:04:07,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-25T12:04:07,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 579 msec 2024-11-25T12:04:07,228 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 622 msec 2024-11-25T12:04:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-25T12:04:07,232 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:07,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb7e9dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,237 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:07,237 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:07,238 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:07,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:07,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:07,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@691cdb71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:07,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:07,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:07,242 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@455bb67e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@486e0901, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,245 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:07,245 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:07,246 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:07,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:07,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:07,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a6df162, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:07,247 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:07,247 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:07,248 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:07,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@512ef9e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,250 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42f18193, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,250 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:07,250 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:07,251 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:07,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b30dd76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:07,253 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:07,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41866eae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cde9120, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,255 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:07,255 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:07,256 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b65fa8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:07,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:07,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:07,258 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:07,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2667302f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a1653a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,260 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:07,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:07,261 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:07,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:07,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:07,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@769ec81d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:07,263 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:07,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d917652, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48fa262a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,267 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:07,267 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:07,268 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79250b30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:07,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:07,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:07,270 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5558d16c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5808d603, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,272 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:07,272 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:07,273 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6746317a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:07,275 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:07,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29db9dab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c960a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,277 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:07,277 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:07,278 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79f5ff67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:07,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:07,280 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:07,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28233673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c277cfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,285 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:07,285 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:07,288 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@200c223b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:07,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:07,291 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:07,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d098d10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:07,292 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:04:07,293 DEBUG [Thread-580 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:07,293 DEBUG [Thread-585 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:07,294 DEBUG [Thread-587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:07,294 DEBUG [Thread-591 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:07,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-25T12:04:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:07,295 DEBUG [Thread-589 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:07,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:07,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T12:04:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:07,297 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:07,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:07,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:07,298 DEBUG [Thread-578 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:07,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:07,299 DEBUG [Thread-576 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:07,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:07,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:07,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:07,300 DEBUG [Thread-576 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:07,301 DEBUG [Thread-576 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:07,301 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:04:07,302 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:07,303 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:04:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:07,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:04:07,303 DEBUG [Thread-574 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:07,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:04:07,304 DEBUG [Thread-582 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:07,304 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:07,305 DEBUG [Thread-582 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:07,305 DEBUG [Thread-582 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:04:07,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:07,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:04:07,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:04:07,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:04:07,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:04:07,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:04:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:07,317 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:07,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:04:07,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:04:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:07,334 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:07,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ed09f1e402e2ed659ccb58744c29bb 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T12:04:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T12:04:07,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:04:07,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:07,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:07,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:07,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:07,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:07,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:07,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112554e363e2e4a3480b9ec4ca92d29b092b_14ed09f1e402e2ed659ccb58744c29bb is 50, key is test_row_0/A:col10/1732536247328/Put/seqid=0 2024-11-25T12:04:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536307444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:07,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,472 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:07,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536307450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:07,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:07,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536307465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:07,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536307469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:07,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:07,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60736 deadline: 1732536307471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:07,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741936_1112 (size=12154) 2024-11-25T12:04:07,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536307576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:07,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:07,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:07,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536307598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:07,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:07,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536307599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:07,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:07,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:07,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536307604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:07,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:04:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T12:04:07,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:07,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:07,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:07,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:07,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:07,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:07,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60736 deadline: 1732536307618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:07,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:04:07,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:07,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:07,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:07,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:07,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:07,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:07,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:07,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536307806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:07,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:07,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536307817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:07,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:07,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536307817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:07,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:07,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536307828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:07,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:07,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:07,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60736 deadline: 1732536307849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:07,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:07,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T12:04:07,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:04:07,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:07,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:07,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:07,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:07,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:07,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:07,943 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:07,971 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112554e363e2e4a3480b9ec4ca92d29b092b_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112554e363e2e4a3480b9ec4ca92d29b092b_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:07,979 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/31c384b96e3f48a1851a02dbcc6a9e7e, store: [table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:08,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/31c384b96e3f48a1851a02dbcc6a9e7e is 175, key is test_row_0/A:col10/1732536247328/Put/seqid=0 2024-11-25T12:04:08,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741937_1113 (size=30955) 2024-11-25T12:04:08,056 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/31c384b96e3f48a1851a02dbcc6a9e7e 2024-11-25T12:04:08,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:04:08,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:08,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:08,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:08,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:08,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:08,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:08,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536308119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:08,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:08,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536308136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:08,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:08,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536308137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:08,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:08,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:08,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536308148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:08,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:08,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/845707e599794c2397d70a77f7638c34 is 50, key is test_row_0/B:col10/1732536247328/Put/seqid=0 2024-11-25T12:04:08,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:08,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60736 deadline: 1732536308169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:08,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:08,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741938_1114 (size=12001) 2024-11-25T12:04:08,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/845707e599794c2397d70a77f7638c34 2024-11-25T12:04:08,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:04:08,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:08,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:08,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:08,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:08,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:08,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:08,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/45dbd651deed41e09bfd1fc55d827ac2 is 50, key is test_row_0/C:col10/1732536247328/Put/seqid=0 2024-11-25T12:04:08,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:04:08,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:08,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:08,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:08,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:08,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:08,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T12:04:08,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741939_1115 (size=12001) 2024-11-25T12:04:08,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/45dbd651deed41e09bfd1fc55d827ac2 2024-11-25T12:04:08,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/31c384b96e3f48a1851a02dbcc6a9e7e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/31c384b96e3f48a1851a02dbcc6a9e7e 2024-11-25T12:04:08,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/31c384b96e3f48a1851a02dbcc6a9e7e, entries=150, sequenceid=16, filesize=30.2 K 2024-11-25T12:04:08,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/845707e599794c2397d70a77f7638c34 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/845707e599794c2397d70a77f7638c34 2024-11-25T12:04:08,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:04:08,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:08,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:08,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:08,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:08,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:08,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:08,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/845707e599794c2397d70a77f7638c34, entries=150, sequenceid=16, filesize=11.7 K 2024-11-25T12:04:08,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/45dbd651deed41e09bfd1fc55d827ac2 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/45dbd651deed41e09bfd1fc55d827ac2 2024-11-25T12:04:08,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/45dbd651deed41e09bfd1fc55d827ac2, entries=150, sequenceid=16, filesize=11.7 K 2024-11-25T12:04:08,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 14ed09f1e402e2ed659ccb58744c29bb in 1290ms, sequenceid=16, compaction requested=false 2024-11-25T12:04:08,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:08,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:08,653 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:08,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ed09f1e402e2ed659ccb58744c29bb 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T12:04:08,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:08,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536308679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:08,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:08,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:08,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536308681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:08,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:08,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536308690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:08,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536308676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:08,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:08,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60736 deadline: 1732536308690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:08,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:08,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:08,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d4fe02c36df34b8696d9a97c5bc2c6de_14ed09f1e402e2ed659ccb58744c29bb is 50, key is test_row_0/A:col10/1732536247454/Put/seqid=0 2024-11-25T12:04:08,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:04:08,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:08,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:08,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:08,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:08,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:08,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:08,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741940_1116 (size=14594) 2024-11-25T12:04:08,758 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:08,765 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d4fe02c36df34b8696d9a97c5bc2c6de_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d4fe02c36df34b8696d9a97c5bc2c6de_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:08,774 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/4829966e3d264b6dafc90d45a35465e8, store: [table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:08,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/4829966e3d264b6dafc90d45a35465e8 is 175, key is test_row_0/A:col10/1732536247454/Put/seqid=0 2024-11-25T12:04:08,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:08,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536308799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:08,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:08,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:08,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536308799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:08,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,802 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:08,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:08,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536308807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:08,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:08,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536308807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:08,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:08,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:08,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:08,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741941_1117 (size=39549) 2024-11-25T12:04:08,822 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/4829966e3d264b6dafc90d45a35465e8 2024-11-25T12:04:08,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/b4e206b0605e4e42ba47213400d77cb5 is 50, key is test_row_0/B:col10/1732536247454/Put/seqid=0 2024-11-25T12:04:08,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741942_1118 (size=12001) 2024-11-25T12:04:08,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/b4e206b0605e4e42ba47213400d77cb5 2024-11-25T12:04:08,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/feb2ac6c5a3e48e2bd36ce6174ea2a6d is 50, key is test_row_0/C:col10/1732536247454/Put/seqid=0 2024-11-25T12:04:08,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:04:08,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:08,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:08,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:08,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:08,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:08,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741943_1119 (size=12001) 2024-11-25T12:04:08,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/feb2ac6c5a3e48e2bd36ce6174ea2a6d 2024-11-25T12:04:08,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/4829966e3d264b6dafc90d45a35465e8 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/4829966e3d264b6dafc90d45a35465e8 2024-11-25T12:04:08,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/4829966e3d264b6dafc90d45a35465e8, entries=200, sequenceid=41, filesize=38.6 K 2024-11-25T12:04:08,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/b4e206b0605e4e42ba47213400d77cb5 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/b4e206b0605e4e42ba47213400d77cb5 2024-11-25T12:04:08,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/b4e206b0605e4e42ba47213400d77cb5, entries=150, sequenceid=41, filesize=11.7 K 2024-11-25T12:04:09,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/feb2ac6c5a3e48e2bd36ce6174ea2a6d as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/feb2ac6c5a3e48e2bd36ce6174ea2a6d 2024-11-25T12:04:09,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536309007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:09,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/feb2ac6c5a3e48e2bd36ce6174ea2a6d, entries=150, sequenceid=41, filesize=11.7 K 2024-11-25T12:04:09,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:09,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 14ed09f1e402e2ed659ccb58744c29bb in 368ms, sequenceid=41, compaction requested=false 2024-11-25T12:04:09,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:09,022 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:09,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ed09f1e402e2ed659ccb58744c29bb 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T12:04:09,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259e2770da8e2a47d9827a033824235965_14ed09f1e402e2ed659ccb58744c29bb is 50, key is test_row_0/A:col10/1732536249016/Put/seqid=0 2024-11-25T12:04:09,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:04:09,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:09,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:09,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:09,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:09,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:09,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741944_1120 (size=14594) 2024-11-25T12:04:09,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536309183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:09,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536309192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:09,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536309197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:09,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:04:09,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:09,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:09,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:09,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:09,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:09,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:09,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536309307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536309307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:09,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:09,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:09,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536309309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:09,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:09,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536309328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:09,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:04:09,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:09,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:09,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:09,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:09,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T12:04:09,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536309516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:09,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:09,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:04:09,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:09,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:09,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:09,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:09,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:09,527 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:09,537 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259e2770da8e2a47d9827a033824235965_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259e2770da8e2a47d9827a033824235965_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:09,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536309527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:09,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:09,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536309528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:09,546 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/8a1d328198ff400fa884667f0cd4917f, store: [table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:09,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:09,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/8a1d328198ff400fa884667f0cd4917f is 175, key is test_row_0/A:col10/1732536249016/Put/seqid=0 2024-11-25T12:04:09,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741945_1121 (size=39549) 2024-11-25T12:04:09,600 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/8a1d328198ff400fa884667f0cd4917f 2024-11-25T12:04:09,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/374c692220c34235aa57fffccccd345a is 50, key is test_row_0/B:col10/1732536249016/Put/seqid=0 2024-11-25T12:04:09,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:04:09,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:09,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:09,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:09,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:09,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:09,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741946_1122 (size=12001) 2024-11-25T12:04:09,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60736 deadline: 1732536309706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:09,707 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_2' in 14ed09f1e402e2ed659ccb58744c29bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:04:09,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:09,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:09,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536309826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:09,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:04:09,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:09,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:09,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:09,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:09,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:09,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:09,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:09,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536309847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:09,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:09,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536309847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:09,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:09,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536309856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:09,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:09,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:10,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:04:10,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:10,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:10,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:10,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:10,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:10,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/374c692220c34235aa57fffccccd345a 2024-11-25T12:04:10,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/426966af2a9540b18d3fbb8bcaf076cd is 50, key is test_row_0/C:col10/1732536249016/Put/seqid=0 2024-11-25T12:04:10,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:04:10,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:10,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:10,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:10,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:10,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:10,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:10,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741947_1123 (size=12001) 2024-11-25T12:04:10,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/426966af2a9540b18d3fbb8bcaf076cd 2024-11-25T12:04:10,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/8a1d328198ff400fa884667f0cd4917f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/8a1d328198ff400fa884667f0cd4917f 2024-11-25T12:04:10,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/8a1d328198ff400fa884667f0cd4917f, entries=200, sequenceid=53, filesize=38.6 K 2024-11-25T12:04:10,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/374c692220c34235aa57fffccccd345a as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/374c692220c34235aa57fffccccd345a 2024-11-25T12:04:10,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/374c692220c34235aa57fffccccd345a, entries=150, sequenceid=53, filesize=11.7 K 2024-11-25T12:04:10,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/426966af2a9540b18d3fbb8bcaf076cd as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/426966af2a9540b18d3fbb8bcaf076cd 2024-11-25T12:04:10,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/426966af2a9540b18d3fbb8bcaf076cd, entries=150, sequenceid=53, filesize=11.7 K 2024-11-25T12:04:10,271 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 14ed09f1e402e2ed659ccb58744c29bb in 1249ms, sequenceid=53, compaction requested=true 2024-11-25T12:04:10,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:10,271 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:10,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:04:10,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:10,272 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:10,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:04:10,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:10,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:04:10,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:10,278 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:10,278 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:10,278 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/B is initiating minor compaction (all files) 2024-11-25T12:04:10,278 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/A is initiating minor compaction (all files) 2024-11-25T12:04:10,278 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/A in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:10,278 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/B in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:10,278 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/31c384b96e3f48a1851a02dbcc6a9e7e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/4829966e3d264b6dafc90d45a35465e8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/8a1d328198ff400fa884667f0cd4917f] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=107.5 K 2024-11-25T12:04:10,278 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/845707e599794c2397d70a77f7638c34, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/b4e206b0605e4e42ba47213400d77cb5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/374c692220c34235aa57fffccccd345a] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=35.2 K 2024-11-25T12:04:10,278 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:10,279 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/31c384b96e3f48a1851a02dbcc6a9e7e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/4829966e3d264b6dafc90d45a35465e8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/8a1d328198ff400fa884667f0cd4917f] 2024-11-25T12:04:10,279 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 845707e599794c2397d70a77f7638c34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732536247328 2024-11-25T12:04:10,280 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting b4e206b0605e4e42ba47213400d77cb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732536247403 2024-11-25T12:04:10,280 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31c384b96e3f48a1851a02dbcc6a9e7e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732536247328 2024-11-25T12:04:10,282 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 374c692220c34235aa57fffccccd345a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732536248678 2024-11-25T12:04:10,282 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4829966e3d264b6dafc90d45a35465e8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732536247403 2024-11-25T12:04:10,286 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a1d328198ff400fa884667f0cd4917f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732536248673 2024-11-25T12:04:10,314 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#B#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:10,314 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/357debbece1c46ceaeb5a2182c0ab622 is 50, key is test_row_0/B:col10/1732536249016/Put/seqid=0 2024-11-25T12:04:10,320 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:10,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-25T12:04:10,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:10,334 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 14ed09f1e402e2ed659ccb58744c29bb 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T12:04:10,345 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112580bd20e2b2434ba18423a45893729d3b_14ed09f1e402e2ed659ccb58744c29bb store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:10,359 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112580bd20e2b2434ba18423a45893729d3b_14ed09f1e402e2ed659ccb58744c29bb, store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:10,359 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112580bd20e2b2434ba18423a45893729d3b_14ed09f1e402e2ed659ccb58744c29bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:10,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:10,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:10,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741948_1124 (size=12104) 2024-11-25T12:04:10,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256651dbfb9d9842b09c4d585a020b56a7_14ed09f1e402e2ed659ccb58744c29bb is 50, key is test_row_0/A:col10/1732536249191/Put/seqid=0 2024-11-25T12:04:10,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741949_1125 (size=4469) 2024-11-25T12:04:10,414 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#A#compaction#106 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:10,415 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/4d92d500a2d04c7faef954fe16e7f63e is 175, key is test_row_0/A:col10/1732536249016/Put/seqid=0 2024-11-25T12:04:10,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536310421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:10,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:10,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741950_1126 (size=12154) 2024-11-25T12:04:10,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536310425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:10,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536310429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:10,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:10,473 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T12:04:10,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741951_1127 (size=31058) 2024-11-25T12:04:10,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536310536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:10,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:10,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:10,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:10,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536310546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:10,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:10,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536310557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:10,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:10,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:10,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:10,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:10,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536310747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:10,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:10,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:10,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:10,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:10,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536310766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:10,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:10,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:10,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536310779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:10,815 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/357debbece1c46ceaeb5a2182c0ab622 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/357debbece1c46ceaeb5a2182c0ab622 2024-11-25T12:04:10,835 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/B of 14ed09f1e402e2ed659ccb58744c29bb into 357debbece1c46ceaeb5a2182c0ab622(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:10,836 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:10,836 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/B, priority=13, startTime=1732536250272; duration=0sec 2024-11-25T12:04:10,836 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:10,836 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:B 2024-11-25T12:04:10,836 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:10,838 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:10,838 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/C is initiating minor compaction (all files) 2024-11-25T12:04:10,839 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/C in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:10,839 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/45dbd651deed41e09bfd1fc55d827ac2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/feb2ac6c5a3e48e2bd36ce6174ea2a6d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/426966af2a9540b18d3fbb8bcaf076cd] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=35.2 K 2024-11-25T12:04:10,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:10,843 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 45dbd651deed41e09bfd1fc55d827ac2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732536247328 2024-11-25T12:04:10,845 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting feb2ac6c5a3e48e2bd36ce6174ea2a6d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732536247403 2024-11-25T12:04:10,847 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 426966af2a9540b18d3fbb8bcaf076cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732536248678 2024-11-25T12:04:10,865 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256651dbfb9d9842b09c4d585a020b56a7_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256651dbfb9d9842b09c4d585a020b56a7_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:10,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/1cfacb0707664d99b07b913ec3350b86, store: [table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:10,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/1cfacb0707664d99b07b913ec3350b86 is 175, key is test_row_0/A:col10/1732536249191/Put/seqid=0 2024-11-25T12:04:10,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:10,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536310870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:10,886 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_1' in 14ed09f1e402e2ed659ccb58744c29bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:04:10,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:10,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:10,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:10,899 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#C#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:10,900 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/347cd8818dec4379a2f0bbc0b27f94f8 is 50, key is test_row_0/C:col10/1732536249016/Put/seqid=0 2024-11-25T12:04:10,914 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/4d92d500a2d04c7faef954fe16e7f63e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/4d92d500a2d04c7faef954fe16e7f63e 2024-11-25T12:04:10,922 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/A of 14ed09f1e402e2ed659ccb58744c29bb into 4d92d500a2d04c7faef954fe16e7f63e(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:10,922 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:10,922 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/A, priority=13, startTime=1732536250271; duration=0sec 2024-11-25T12:04:10,923 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:10,923 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:A 2024-11-25T12:04:10,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741952_1128 (size=30955) 2024-11-25T12:04:10,970 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/1cfacb0707664d99b07b913ec3350b86 2024-11-25T12:04:10,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741953_1129 (size=12104) 2024-11-25T12:04:11,004 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/347cd8818dec4379a2f0bbc0b27f94f8 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/347cd8818dec4379a2f0bbc0b27f94f8 2024-11-25T12:04:11,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/f88c1e824b3a434e86a8c8e971ae1403 is 50, key is test_row_0/B:col10/1732536249191/Put/seqid=0 2024-11-25T12:04:11,029 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/C of 14ed09f1e402e2ed659ccb58744c29bb into 347cd8818dec4379a2f0bbc0b27f94f8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:11,029 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:11,029 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/C, priority=13, startTime=1732536250276; duration=0sec 2024-11-25T12:04:11,029 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:11,029 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:C 2024-11-25T12:04:11,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741954_1130 (size=12001) 2024-11-25T12:04:11,060 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/f88c1e824b3a434e86a8c8e971ae1403 2024-11-25T12:04:11,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536311066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:11,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:11,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/61f2e05c6d75485eb1a4e094573a2180 is 50, key is test_row_0/C:col10/1732536249191/Put/seqid=0 2024-11-25T12:04:11,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536311086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:11,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:11,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:11,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:11,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536311105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:11,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741955_1131 (size=12001) 2024-11-25T12:04:11,162 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/61f2e05c6d75485eb1a4e094573a2180 2024-11-25T12:04:11,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/1cfacb0707664d99b07b913ec3350b86 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/1cfacb0707664d99b07b913ec3350b86 2024-11-25T12:04:11,195 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/1cfacb0707664d99b07b913ec3350b86, entries=150, sequenceid=77, filesize=30.2 K 2024-11-25T12:04:11,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/f88c1e824b3a434e86a8c8e971ae1403 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/f88c1e824b3a434e86a8c8e971ae1403 2024-11-25T12:04:11,208 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/f88c1e824b3a434e86a8c8e971ae1403, entries=150, sequenceid=77, filesize=11.7 K 2024-11-25T12:04:11,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/61f2e05c6d75485eb1a4e094573a2180 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/61f2e05c6d75485eb1a4e094573a2180 2024-11-25T12:04:11,222 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/61f2e05c6d75485eb1a4e094573a2180, entries=150, sequenceid=77, filesize=11.7 K 2024-11-25T12:04:11,224 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 14ed09f1e402e2ed659ccb58744c29bb in 890ms, sequenceid=77, compaction requested=false 2024-11-25T12:04:11,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:11,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:11,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-25T12:04:11,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-25T12:04:11,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-25T12:04:11,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.9340 sec 2024-11-25T12:04:11,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 3.9580 sec 2024-11-25T12:04:11,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-25T12:04:11,467 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:11,468 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:04:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-25T12:04:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T12:04:11,472 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:04:11,472 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:04:11,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:04:11,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T12:04:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:11,578 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:11,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ed09f1e402e2ed659ccb58744c29bb 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-25T12:04:11,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112582bae34884f34a90b5c682d0d2616b16_14ed09f1e402e2ed659ccb58744c29bb is 50, key is test_row_0/A:col10/1732536251576/Put/seqid=0 2024-11-25T12:04:11,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:04:11,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:11,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:11,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:11,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:11,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:11,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741956_1132 (size=12154) 2024-11-25T12:04:11,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:11,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536311684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:11,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:11,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:11,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:11,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:11,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536311685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:11,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:11,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:11,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:11,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536311688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:11,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:11,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:11,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:11,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:11,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60736 deadline: 1732536311726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:11,732 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_2' in 14ed09f1e402e2ed659ccb58744c29bb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:04:11,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:11,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:11,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T12:04:11,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:04:11,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:11,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:11,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:11,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:11,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:11,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:11,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536311797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:11,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:11,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:11,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536311797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:11,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:11,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:11,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:11,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536311797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:11,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:04:11,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:11,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:11,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:11,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:11,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:11,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:12,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:12,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536312007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:12,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:12,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536312007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:12,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:12,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536312007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:12,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:12,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:12,056 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:12,063 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112582bae34884f34a90b5c682d0d2616b16_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112582bae34884f34a90b5c682d0d2616b16_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:12,064 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/f1e3cf7d7b694e32a462cb441a82fcb4, store: [table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:12,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/f1e3cf7d7b694e32a462cb441a82fcb4 is 175, key is test_row_0/A:col10/1732536251576/Put/seqid=0 2024-11-25T12:04:12,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741957_1133 (size=30955) 2024-11-25T12:04:12,083 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/f1e3cf7d7b694e32a462cb441a82fcb4 2024-11-25T12:04:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T12:04:12,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/09ea3eb0705e4ecf82d0592d0f737d88 is 50, key is test_row_0/B:col10/1732536251576/Put/seqid=0 2024-11-25T12:04:12,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:04:12,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:12,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:12,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:12,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:12,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:12,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741958_1134 (size=12001) 2024-11-25T12:04:12,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/09ea3eb0705e4ecf82d0592d0f737d88 2024-11-25T12:04:12,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/3bb02f08f06d45ed9777f7aa4f732fa8 is 50, key is test_row_0/C:col10/1732536251576/Put/seqid=0 2024-11-25T12:04:12,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741959_1135 (size=12001) 2024-11-25T12:04:12,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/3bb02f08f06d45ed9777f7aa4f732fa8 2024-11-25T12:04:12,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/f1e3cf7d7b694e32a462cb441a82fcb4 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/f1e3cf7d7b694e32a462cb441a82fcb4 2024-11-25T12:04:12,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/f1e3cf7d7b694e32a462cb441a82fcb4, entries=150, sequenceid=93, filesize=30.2 K 2024-11-25T12:04:12,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/09ea3eb0705e4ecf82d0592d0f737d88 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/09ea3eb0705e4ecf82d0592d0f737d88 2024-11-25T12:04:12,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:04:12,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:12,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:12,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:12,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:12,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:12,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/09ea3eb0705e4ecf82d0592d0f737d88, entries=150, sequenceid=93, filesize=11.7 K 2024-11-25T12:04:12,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/3bb02f08f06d45ed9777f7aa4f732fa8 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/3bb02f08f06d45ed9777f7aa4f732fa8 2024-11-25T12:04:12,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/3bb02f08f06d45ed9777f7aa4f732fa8, entries=150, sequenceid=93, filesize=11.7 K 2024-11-25T12:04:12,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 14ed09f1e402e2ed659ccb58744c29bb in 715ms, sequenceid=93, compaction requested=true 2024-11-25T12:04:12,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:12,294 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:12,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:04:12,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:12,295 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:12,295 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/A is initiating minor compaction (all files) 2024-11-25T12:04:12,295 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/A in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:12,295 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/4d92d500a2d04c7faef954fe16e7f63e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/1cfacb0707664d99b07b913ec3350b86, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/f1e3cf7d7b694e32a462cb441a82fcb4] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=90.8 K 2024-11-25T12:04:12,295 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:12,295 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:12,295 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/4d92d500a2d04c7faef954fe16e7f63e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/1cfacb0707664d99b07b913ec3350b86, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/f1e3cf7d7b694e32a462cb441a82fcb4] 2024-11-25T12:04:12,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:04:12,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:12,296 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d92d500a2d04c7faef954fe16e7f63e, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732536248678 2024-11-25T12:04:12,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:04:12,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:12,297 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cfacb0707664d99b07b913ec3350b86, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732536249172 2024-11-25T12:04:12,297 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:12,297 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/B is initiating minor compaction (all files) 2024-11-25T12:04:12,297 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/B in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:12,297 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/357debbece1c46ceaeb5a2182c0ab622, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/f88c1e824b3a434e86a8c8e971ae1403, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/09ea3eb0705e4ecf82d0592d0f737d88] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=35.3 K 2024-11-25T12:04:12,298 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 357debbece1c46ceaeb5a2182c0ab622, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732536248678 2024-11-25T12:04:12,298 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1e3cf7d7b694e32a462cb441a82fcb4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732536250410 2024-11-25T12:04:12,306 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting f88c1e824b3a434e86a8c8e971ae1403, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732536249172 2024-11-25T12:04:12,307 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 09ea3eb0705e4ecf82d0592d0f737d88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732536250410 2024-11-25T12:04:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:12,321 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:12,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ed09f1e402e2ed659ccb58744c29bb 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T12:04:12,323 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:12,327 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112531ce97b0be7f416da1dae99bb2abdaa7_14ed09f1e402e2ed659ccb58744c29bb store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:12,329 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#B#compaction#115 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:12,330 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/002a10d0e2d9482299ecca0b37ff7fbe is 50, key is test_row_0/B:col10/1732536251576/Put/seqid=0 2024-11-25T12:04:12,331 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112531ce97b0be7f416da1dae99bb2abdaa7_14ed09f1e402e2ed659ccb58744c29bb, store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:12,331 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112531ce97b0be7f416da1dae99bb2abdaa7_14ed09f1e402e2ed659ccb58744c29bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:12,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:12,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536312357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:12,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e22c6fb6523048e8b6d460523d81022e_14ed09f1e402e2ed659ccb58744c29bb is 50, key is test_row_0/A:col10/1732536251679/Put/seqid=0 2024-11-25T12:04:12,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:12,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:12,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536312359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:12,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536312373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:12,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741960_1136 (size=4469) 2024-11-25T12:04:12,384 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#A#compaction#114 average throughput is 0.40 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:12,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:12,385 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/bab8f05ab95d46a89d4ef0ea9bdaf140 is 175, key is test_row_0/A:col10/1732536251576/Put/seqid=0 2024-11-25T12:04:12,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741961_1137 (size=12207) 2024-11-25T12:04:12,403 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/002a10d0e2d9482299ecca0b37ff7fbe as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/002a10d0e2d9482299ecca0b37ff7fbe 2024-11-25T12:04:12,411 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/B of 14ed09f1e402e2ed659ccb58744c29bb into 002a10d0e2d9482299ecca0b37ff7fbe(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:12,411 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:12,411 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/B, priority=13, startTime=1732536252295; duration=0sec 2024-11-25T12:04:12,411 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:12,411 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:B 2024-11-25T12:04:12,411 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:12,412 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:12,412 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/C is initiating minor compaction (all files) 2024-11-25T12:04:12,412 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/C in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:12,413 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/347cd8818dec4379a2f0bbc0b27f94f8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/61f2e05c6d75485eb1a4e094573a2180, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/3bb02f08f06d45ed9777f7aa4f732fa8] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=35.3 K 2024-11-25T12:04:12,413 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 347cd8818dec4379a2f0bbc0b27f94f8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732536248678 2024-11-25T12:04:12,413 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 61f2e05c6d75485eb1a4e094573a2180, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732536249172 2024-11-25T12:04:12,414 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bb02f08f06d45ed9777f7aa4f732fa8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732536250410 2024-11-25T12:04:12,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:04:12,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:12,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:12,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:12,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:12,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:12,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741962_1138 (size=14594) 2024-11-25T12:04:12,438 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:12,449 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e22c6fb6523048e8b6d460523d81022e_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e22c6fb6523048e8b6d460523d81022e_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:12,451 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/47b9615aa4634ce8ab178e6b50992dc7, store: [table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:12,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/47b9615aa4634ce8ab178e6b50992dc7 is 175, key is test_row_0/A:col10/1732536251679/Put/seqid=0 2024-11-25T12:04:12,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741963_1139 (size=31161) 2024-11-25T12:04:12,468 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#C#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:12,469 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/b6ce9d610a5841c18b2f6f4509f03a5d is 50, key is test_row_0/C:col10/1732536251576/Put/seqid=0 2024-11-25T12:04:12,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:12,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536312477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:12,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536312481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:12,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:12,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536312507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:12,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:12,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741964_1140 (size=39549) 2024-11-25T12:04:12,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741965_1141 (size=12207) 2024-11-25T12:04:12,567 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/b6ce9d610a5841c18b2f6f4509f03a5d as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/b6ce9d610a5841c18b2f6f4509f03a5d 2024-11-25T12:04:12,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:04:12,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:12,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:12,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:12,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:12,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:12,578 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/C of 14ed09f1e402e2ed659ccb58744c29bb into b6ce9d610a5841c18b2f6f4509f03a5d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:12,578 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:12,579 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/C, priority=13, startTime=1732536252296; duration=0sec 2024-11-25T12:04:12,579 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:12,579 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:C 2024-11-25T12:04:12,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T12:04:12,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536312696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:12,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:12,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536312696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:12,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:12,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:12,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536312717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:12,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:12,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:04:12,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:12,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:12,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:12,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:12,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:12,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:04:12,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:12,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:12,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:12,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:12,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:12,900 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/bab8f05ab95d46a89d4ef0ea9bdaf140 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/bab8f05ab95d46a89d4ef0ea9bdaf140 2024-11-25T12:04:12,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:12,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:12,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536312909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:12,916 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/A of 14ed09f1e402e2ed659ccb58744c29bb into bab8f05ab95d46a89d4ef0ea9bdaf140(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:12,916 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:12,916 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/A, priority=13, startTime=1732536252294; duration=0sec 2024-11-25T12:04:12,916 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:12,916 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:A 2024-11-25T12:04:12,918 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_1' in 14ed09f1e402e2ed659ccb58744c29bb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:04:12,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:12,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:12,921 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/47b9615aa4634ce8ab178e6b50992dc7 2024-11-25T12:04:12,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/bc97985f7f9b411ab28f1c3be1eb8368 is 50, key is test_row_0/B:col10/1732536251679/Put/seqid=0 2024-11-25T12:04:12,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741966_1142 (size=12001) 2024-11-25T12:04:12,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/bc97985f7f9b411ab28f1c3be1eb8368 2024-11-25T12:04:13,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/a5ea9abe76034f7c81c2b1494acaf4ef is 50, key is test_row_0/C:col10/1732536251679/Put/seqid=0 2024-11-25T12:04:13,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:13,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536313009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:13,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:13,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536313009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:13,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:13,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:13,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:13,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:13,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536313027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:13,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:13,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741967_1143 (size=12001) 2024-11-25T12:04:13,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:04:13,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:13,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:13,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:13,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:13,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:13,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:13,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:04:13,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:13,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:13,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:13,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:13,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:13,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:13,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:04:13,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:13,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:13,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:13,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:13,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:13,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:13,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/a5ea9abe76034f7c81c2b1494acaf4ef 2024-11-25T12:04:13,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/47b9615aa4634ce8ab178e6b50992dc7 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/47b9615aa4634ce8ab178e6b50992dc7 2024-11-25T12:04:13,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/47b9615aa4634ce8ab178e6b50992dc7, entries=200, sequenceid=117, filesize=38.6 K 2024-11-25T12:04:13,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/bc97985f7f9b411ab28f1c3be1eb8368 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/bc97985f7f9b411ab28f1c3be1eb8368 2024-11-25T12:04:13,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:04:13,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:13,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:13,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:13,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:13,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:13,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/bc97985f7f9b411ab28f1c3be1eb8368, entries=150, sequenceid=117, filesize=11.7 K 2024-11-25T12:04:13,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536313527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:13,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:13,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:13,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:13,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536313529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:13,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:13,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:13,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:13,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:13,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536313549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:13,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:13,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:13,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:13,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/a5ea9abe76034f7c81c2b1494acaf4ef as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/a5ea9abe76034f7c81c2b1494acaf4ef 2024-11-25T12:04:13,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/a5ea9abe76034f7c81c2b1494acaf4ef, entries=150, sequenceid=117, filesize=11.7 K 2024-11-25T12:04:13,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 14ed09f1e402e2ed659ccb58744c29bb in 1255ms, sequenceid=117, compaction requested=false 2024-11-25T12:04:13,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T12:04:13,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-25T12:04:13,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:13,686 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 14ed09f1e402e2ed659ccb58744c29bb 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T12:04:13,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ae9661048b3b41fcbc5dc07337bbfd96_14ed09f1e402e2ed659ccb58744c29bb is 50, key is test_row_0/A:col10/1732536252357/Put/seqid=0 2024-11-25T12:04:13,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741968_1144 (size=12154) 2024-11-25T12:04:13,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:13,818 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ae9661048b3b41fcbc5dc07337bbfd96_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ae9661048b3b41fcbc5dc07337bbfd96_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:13,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/dc86c4a9b0824213a7c5e61672bef541, store: [table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:13,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/dc86c4a9b0824213a7c5e61672bef541 is 175, key is test_row_0/A:col10/1732536252357/Put/seqid=0 2024-11-25T12:04:13,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741969_1145 (size=30955) 2024-11-25T12:04:13,881 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/dc86c4a9b0824213a7c5e61672bef541 2024-11-25T12:04:13,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/f7e1ca5596104e95ba4629c28370a9d4 is 50, key is test_row_0/B:col10/1732536252357/Put/seqid=0 2024-11-25T12:04:13,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741970_1146 (size=12001) 2024-11-25T12:04:13,992 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/f7e1ca5596104e95ba4629c28370a9d4 2024-11-25T12:04:14,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/32bd7d06a66e413297100e1b7e7d1512 is 50, key is test_row_0/C:col10/1732536252357/Put/seqid=0 2024-11-25T12:04:14,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741971_1147 (size=12001) 2024-11-25T12:04:14,482 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/32bd7d06a66e413297100e1b7e7d1512 2024-11-25T12:04:14,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/dc86c4a9b0824213a7c5e61672bef541 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/dc86c4a9b0824213a7c5e61672bef541 2024-11-25T12:04:14,512 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/dc86c4a9b0824213a7c5e61672bef541, entries=150, sequenceid=132, filesize=30.2 K 2024-11-25T12:04:14,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/f7e1ca5596104e95ba4629c28370a9d4 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/f7e1ca5596104e95ba4629c28370a9d4 2024-11-25T12:04:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,534 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/f7e1ca5596104e95ba4629c28370a9d4, entries=150, sequenceid=132, filesize=11.7 K 2024-11-25T12:04:14,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/32bd7d06a66e413297100e1b7e7d1512 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/32bd7d06a66e413297100e1b7e7d1512 2024-11-25T12:04:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,550 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/32bd7d06a66e413297100e1b7e7d1512, entries=150, sequenceid=132, filesize=11.7 K 2024-11-25T12:04:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,552 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for 14ed09f1e402e2ed659ccb58744c29bb in 867ms, sequenceid=132, compaction requested=true 2024-11-25T12:04:14,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:14,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:14,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-25T12:04:14,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-25T12:04:14,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,559 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-25T12:04:14,560 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0830 sec 2024-11-25T12:04:14,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,562 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 3.0920 sec 2024-11-25T12:04:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,848 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:14,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ed09f1e402e2ed659ccb58744c29bb 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:04:14,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112585f9c85e8c704299a3d6624a37935153_14ed09f1e402e2ed659ccb58744c29bb is 50, key is test_row_0/A:col10/1732536254846/Put/seqid=0 2024-11-25T12:04:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741973_1149 (size=27248) 2024-11-25T12:04:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,976 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,982 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112585f9c85e8c704299a3d6624a37935153_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112585f9c85e8c704299a3d6624a37935153_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:14,983 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/ce30aa8125004788b348465b47e3f4b3, store: [table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/ce30aa8125004788b348465b47e3f4b3 is 175, key is test_row_0/A:col10/1732536254846/Put/seqid=0 2024-11-25T12:04:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741972_1148 (size=83031) 2024-11-25T12:04:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:15,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536315152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:15,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:15,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536315157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536315156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:15,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:15,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:15,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536315268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:15,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:15,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536315276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:15,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:15,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536315280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:15,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:15,401 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=144, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/ce30aa8125004788b348465b47e3f4b3 2024-11-25T12:04:15,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/b5e963689e364122b484f11eabe80f21 is 50, key is test_row_0/B:col10/1732536254846/Put/seqid=0 2024-11-25T12:04:15,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536315477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:15,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536315486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:15,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:15,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:15,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:15,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536315486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:15,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:15,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741974_1150 (size=9757) 2024-11-25T12:04:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-25T12:04:15,641 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:15,642 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:04:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-25T12:04:15,644 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:04:15,646 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:04:15,646 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:04:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T12:04:15,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T12:04:15,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60736 deadline: 1732536315784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:15,790 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_2' in 14ed09f1e402e2ed659ccb58744c29bb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8320 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:04:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:15,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:15,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536315797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:15,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:15,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536315797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:15,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:15,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-25T12:04:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:15,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:15,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:15,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:15,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:15,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:15,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:15,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536315800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:15,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/b5e963689e364122b484f11eabe80f21 2024-11-25T12:04:15,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/fd84a9ce512440bc968b729430d33754 is 50, key is test_row_0/C:col10/1732536254846/Put/seqid=0 2024-11-25T12:04:15,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-25T12:04:15,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T12:04:15,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:15,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:15,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:15,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:15,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:15,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741975_1151 (size=9757) 2024-11-25T12:04:15,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/fd84a9ce512440bc968b729430d33754 2024-11-25T12:04:16,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/ce30aa8125004788b348465b47e3f4b3 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/ce30aa8125004788b348465b47e3f4b3 2024-11-25T12:04:16,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/ce30aa8125004788b348465b47e3f4b3, entries=450, sequenceid=144, filesize=81.1 K 2024-11-25T12:04:16,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/b5e963689e364122b484f11eabe80f21 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/b5e963689e364122b484f11eabe80f21 2024-11-25T12:04:16,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/b5e963689e364122b484f11eabe80f21, entries=100, sequenceid=144, filesize=9.5 K 2024-11-25T12:04:16,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/fd84a9ce512440bc968b729430d33754 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/fd84a9ce512440bc968b729430d33754 2024-11-25T12:04:16,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/fd84a9ce512440bc968b729430d33754, entries=100, sequenceid=144, filesize=9.5 K 2024-11-25T12:04:16,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 14ed09f1e402e2ed659ccb58744c29bb in 1208ms, sequenceid=144, compaction requested=true 2024-11-25T12:04:16,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:16,056 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:04:16,063 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 184696 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:04:16,063 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/A is initiating minor compaction (all files) 2024-11-25T12:04:16,063 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/A in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:16,063 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/bab8f05ab95d46a89d4ef0ea9bdaf140, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/47b9615aa4634ce8ab178e6b50992dc7, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/dc86c4a9b0824213a7c5e61672bef541, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/ce30aa8125004788b348465b47e3f4b3] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=180.4 K 2024-11-25T12:04:16,063 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:16,064 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/bab8f05ab95d46a89d4ef0ea9bdaf140, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/47b9615aa4634ce8ab178e6b50992dc7, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/dc86c4a9b0824213a7c5e61672bef541, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/ce30aa8125004788b348465b47e3f4b3] 2024-11-25T12:04:16,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:04:16,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:16,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:04:16,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:16,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:04:16,065 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:04:16,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:16,065 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting bab8f05ab95d46a89d4ef0ea9bdaf140, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732536250410 2024-11-25T12:04:16,073 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47b9615aa4634ce8ab178e6b50992dc7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732536251639 2024-11-25T12:04:16,073 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc86c4a9b0824213a7c5e61672bef541, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536252341 2024-11-25T12:04:16,074 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45966 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:04:16,074 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/B is initiating minor compaction (all files) 2024-11-25T12:04:16,074 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/B in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:16,074 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/002a10d0e2d9482299ecca0b37ff7fbe, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/bc97985f7f9b411ab28f1c3be1eb8368, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/f7e1ca5596104e95ba4629c28370a9d4, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/b5e963689e364122b484f11eabe80f21] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=44.9 K 2024-11-25T12:04:16,075 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce30aa8125004788b348465b47e3f4b3, keycount=450, bloomtype=ROW, size=81.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732536254676 2024-11-25T12:04:16,075 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 002a10d0e2d9482299ecca0b37ff7fbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732536250410 2024-11-25T12:04:16,076 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting bc97985f7f9b411ab28f1c3be1eb8368, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732536251639 2024-11-25T12:04:16,077 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting f7e1ca5596104e95ba4629c28370a9d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536252341 2024-11-25T12:04:16,078 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting b5e963689e364122b484f11eabe80f21, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732536254842 2024-11-25T12:04:16,121 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:16,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-25T12:04:16,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:16,138 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 14ed09f1e402e2ed659ccb58744c29bb 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T12:04:16,153 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125711d3ee97aa040e7b337e94b33fd4d98_14ed09f1e402e2ed659ccb58744c29bb store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:16,159 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125711d3ee97aa040e7b337e94b33fd4d98_14ed09f1e402e2ed659ccb58744c29bb, store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:16,159 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125711d3ee97aa040e7b337e94b33fd4d98_14ed09f1e402e2ed659ccb58744c29bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:16,173 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#B#compaction#127 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:16,174 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/571e75caa7d9499e89d52dc53f391fe8 is 50, key is test_row_0/B:col10/1732536254846/Put/seqid=0 2024-11-25T12:04:16,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d453bb53f02d46e9b9207182132626b3_14ed09f1e402e2ed659ccb58744c29bb is 50, key is test_row_0/A:col10/1732536255152/Put/seqid=0 2024-11-25T12:04:16,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741976_1152 (size=4469) 2024-11-25T12:04:16,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T12:04:16,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741977_1153 (size=12443) 2024-11-25T12:04:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:16,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:16,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741978_1154 (size=12304) 2024-11-25T12:04:16,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:16,349 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/571e75caa7d9499e89d52dc53f391fe8 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/571e75caa7d9499e89d52dc53f391fe8 2024-11-25T12:04:16,359 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d453bb53f02d46e9b9207182132626b3_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d453bb53f02d46e9b9207182132626b3_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:16,363 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/B of 14ed09f1e402e2ed659ccb58744c29bb into 571e75caa7d9499e89d52dc53f391fe8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:16,363 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:16,363 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/B, priority=12, startTime=1732536256064; duration=0sec 2024-11-25T12:04:16,363 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:16,363 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:B 2024-11-25T12:04:16,363 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:04:16,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/63175b59da7b451fb6098f1fd4e70ea8, store: [table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:16,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/63175b59da7b451fb6098f1fd4e70ea8 is 175, key is test_row_0/A:col10/1732536255152/Put/seqid=0 2024-11-25T12:04:16,367 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45966 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:04:16,367 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/C is initiating minor compaction (all files) 2024-11-25T12:04:16,367 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/C in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:16,367 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/b6ce9d610a5841c18b2f6f4509f03a5d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/a5ea9abe76034f7c81c2b1494acaf4ef, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/32bd7d06a66e413297100e1b7e7d1512, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/fd84a9ce512440bc968b729430d33754] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=44.9 K 2024-11-25T12:04:16,368 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting b6ce9d610a5841c18b2f6f4509f03a5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732536250410 2024-11-25T12:04:16,368 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting a5ea9abe76034f7c81c2b1494acaf4ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732536251639 2024-11-25T12:04:16,369 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 32bd7d06a66e413297100e1b7e7d1512, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536252341 2024-11-25T12:04:16,369 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting fd84a9ce512440bc968b729430d33754, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732536254842 2024-11-25T12:04:16,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536316368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:16,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:16,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536316373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:16,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:16,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536316376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:16,404 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#C#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:16,413 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/2564ae70ce394e86a59dec8616bf63b5 is 50, key is test_row_0/C:col10/1732536254846/Put/seqid=0 2024-11-25T12:04:16,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741979_1155 (size=31105) 2024-11-25T12:04:16,431 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/63175b59da7b451fb6098f1fd4e70ea8 2024-11-25T12:04:16,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/7f86f6486147481dac49702f3dc11c03 is 50, key is test_row_0/B:col10/1732536255152/Put/seqid=0 2024-11-25T12:04:16,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741980_1156 (size=12443) 2024-11-25T12:04:16,502 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/2564ae70ce394e86a59dec8616bf63b5 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/2564ae70ce394e86a59dec8616bf63b5 2024-11-25T12:04:16,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:16,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:16,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536316497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:16,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536316497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:16,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:16,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536316496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:16,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:16,511 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/C of 14ed09f1e402e2ed659ccb58744c29bb into 2564ae70ce394e86a59dec8616bf63b5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:16,511 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:16,511 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/C, priority=12, startTime=1732536256065; duration=0sec 2024-11-25T12:04:16,512 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:16,512 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:C 2024-11-25T12:04:16,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741981_1157 (size=12151) 2024-11-25T12:04:16,534 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/7f86f6486147481dac49702f3dc11c03 2024-11-25T12:04:16,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/6384c21f767446a8a5f3e5e872bee1c5 is 50, key is test_row_0/C:col10/1732536255152/Put/seqid=0 2024-11-25T12:04:16,640 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#A#compaction#126 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:16,640 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/87a4228dade2496e84d3d244279140e4 is 175, key is test_row_0/A:col10/1732536254846/Put/seqid=0 2024-11-25T12:04:16,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741982_1158 (size=12151) 2024-11-25T12:04:16,662 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/6384c21f767446a8a5f3e5e872bee1c5 2024-11-25T12:04:16,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741983_1159 (size=31504) 2024-11-25T12:04:16,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/63175b59da7b451fb6098f1fd4e70ea8 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/63175b59da7b451fb6098f1fd4e70ea8 2024-11-25T12:04:16,693 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/87a4228dade2496e84d3d244279140e4 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/87a4228dade2496e84d3d244279140e4 2024-11-25T12:04:16,703 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/63175b59da7b451fb6098f1fd4e70ea8, entries=150, sequenceid=169, filesize=30.4 K 2024-11-25T12:04:16,704 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/A of 14ed09f1e402e2ed659ccb58744c29bb into 87a4228dade2496e84d3d244279140e4(size=30.8 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:16,704 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:16,704 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/A, priority=12, startTime=1732536256056; duration=0sec 2024-11-25T12:04:16,706 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:16,706 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:A 2024-11-25T12:04:16,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/7f86f6486147481dac49702f3dc11c03 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/7f86f6486147481dac49702f3dc11c03 2024-11-25T12:04:16,714 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/7f86f6486147481dac49702f3dc11c03, entries=150, sequenceid=169, filesize=11.9 K 2024-11-25T12:04:16,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/6384c21f767446a8a5f3e5e872bee1c5 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/6384c21f767446a8a5f3e5e872bee1c5 2024-11-25T12:04:16,723 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/6384c21f767446a8a5f3e5e872bee1c5, entries=150, sequenceid=169, filesize=11.9 K 2024-11-25T12:04:16,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:16,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536316716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:16,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:16,729 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 14ed09f1e402e2ed659ccb58744c29bb in 590ms, sequenceid=169, compaction requested=false 2024-11-25T12:04:16,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:16,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:16,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-25T12:04:16,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-25T12:04:16,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-25T12:04:16,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0880 sec 2024-11-25T12:04:16,738 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:16,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ed09f1e402e2ed659ccb58744c29bb 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-25T12:04:16,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:16,740 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.0950 sec 2024-11-25T12:04:16,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125734aa8d7de1b479bb8d02b9734f78c99_14ed09f1e402e2ed659ccb58744c29bb is 50, key is test_row_0/A:col10/1732536256730/Put/seqid=0 2024-11-25T12:04:16,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-25T12:04:16,796 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:16,797 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:04:16,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-25T12:04:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T12:04:16,804 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:04:16,805 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:04:16,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:04:16,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741984_1160 (size=12304) 2024-11-25T12:04:16,826 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:16,852 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125734aa8d7de1b479bb8d02b9734f78c99_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125734aa8d7de1b479bb8d02b9734f78c99_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:16,859 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/afbe494045d84de0873f7fb110e05b36, store: [table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:16,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/afbe494045d84de0873f7fb110e05b36 is 175, key is test_row_0/A:col10/1732536256730/Put/seqid=0 2024-11-25T12:04:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T12:04:16,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741985_1161 (size=31105) 2024-11-25T12:04:16,941 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=185, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/afbe494045d84de0873f7fb110e05b36 2024-11-25T12:04:16,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T12:04:16,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:16,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:16,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:16,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:16,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:16,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:16,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536316966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:16,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:16,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:16,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:16,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/7c8e3e7e730c4ef383e371c0333abaad is 50, key is test_row_0/B:col10/1732536256730/Put/seqid=0 2024-11-25T12:04:16,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536316972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:16,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536316974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:17,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741986_1162 (size=12151) 2024-11-25T12:04:17,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/7c8e3e7e730c4ef383e371c0333abaad 2024-11-25T12:04:17,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536317038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:17,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/03c6d5d8b31942f98e98c07e23373f0a is 50, key is test_row_0/C:col10/1732536256730/Put/seqid=0 2024-11-25T12:04:17,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536317097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:17,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536317097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:17,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:17,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536317101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:17,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:17,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T12:04:17,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T12:04:17,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:17,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:17,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:17,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:17,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:17,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:17,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741987_1163 (size=12151) 2024-11-25T12:04:17,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/03c6d5d8b31942f98e98c07e23373f0a 2024-11-25T12:04:17,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/afbe494045d84de0873f7fb110e05b36 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/afbe494045d84de0873f7fb110e05b36 2024-11-25T12:04:17,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/afbe494045d84de0873f7fb110e05b36, entries=150, sequenceid=185, filesize=30.4 K 2024-11-25T12:04:17,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/7c8e3e7e730c4ef383e371c0333abaad as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/7c8e3e7e730c4ef383e371c0333abaad 2024-11-25T12:04:17,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/7c8e3e7e730c4ef383e371c0333abaad, entries=150, sequenceid=185, filesize=11.9 K 2024-11-25T12:04:17,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/03c6d5d8b31942f98e98c07e23373f0a as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/03c6d5d8b31942f98e98c07e23373f0a 2024-11-25T12:04:17,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/03c6d5d8b31942f98e98c07e23373f0a, entries=150, sequenceid=185, filesize=11.9 K 2024-11-25T12:04:17,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 14ed09f1e402e2ed659ccb58744c29bb in 515ms, sequenceid=185, compaction requested=true 2024-11-25T12:04:17,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:17,253 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:17,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:04:17,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:17,254 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:17,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:04:17,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:17,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:04:17,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:17,258 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93714 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:17,258 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/A is initiating minor compaction (all files) 2024-11-25T12:04:17,259 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/A in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:17,259 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/87a4228dade2496e84d3d244279140e4, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/63175b59da7b451fb6098f1fd4e70ea8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/afbe494045d84de0873f7fb110e05b36] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=91.5 K 2024-11-25T12:04:17,259 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:17,259 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/87a4228dade2496e84d3d244279140e4, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/63175b59da7b451fb6098f1fd4e70ea8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/afbe494045d84de0873f7fb110e05b36] 2024-11-25T12:04:17,259 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36745 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:17,259 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/B is initiating minor compaction (all files) 2024-11-25T12:04:17,259 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/B in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:17,259 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/571e75caa7d9499e89d52dc53f391fe8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/7f86f6486147481dac49702f3dc11c03, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/7c8e3e7e730c4ef383e371c0333abaad] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=35.9 K 2024-11-25T12:04:17,266 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87a4228dade2496e84d3d244279140e4, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732536252357 2024-11-25T12:04:17,267 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63175b59da7b451fb6098f1fd4e70ea8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732536255130 2024-11-25T12:04:17,269 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 571e75caa7d9499e89d52dc53f391fe8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732536252357 2024-11-25T12:04:17,269 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting afbe494045d84de0873f7fb110e05b36, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732536256369 2024-11-25T12:04:17,271 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f86f6486147481dac49702f3dc11c03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732536255130 2024-11-25T12:04:17,271 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c8e3e7e730c4ef383e371c0333abaad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732536256369 2024-11-25T12:04:17,281 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:17,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-25T12:04:17,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:17,305 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411251ee2f911be0b477ba2fd5dd1714d4b00_14ed09f1e402e2ed659ccb58744c29bb store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:17,306 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#B#compaction#136 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:17,306 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/72608898eb2b4db4981fa0ef4fc6f01b is 50, key is test_row_0/B:col10/1732536256730/Put/seqid=0 2024-11-25T12:04:17,307 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 14ed09f1e402e2ed659ccb58744c29bb 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T12:04:17,307 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411251ee2f911be0b477ba2fd5dd1714d4b00_14ed09f1e402e2ed659ccb58744c29bb, store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:17,307 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251ee2f911be0b477ba2fd5dd1714d4b00_14ed09f1e402e2ed659ccb58744c29bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:17,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:17,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:17,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741988_1164 (size=4469) 2024-11-25T12:04:17,358 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#A#compaction#135 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:17,359 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/3a245cc4528b40aca2d2625b46acd7a9 is 175, key is test_row_0/A:col10/1732536256730/Put/seqid=0 2024-11-25T12:04:17,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c7a06c9b1abb4126bb52048be14658ad_14ed09f1e402e2ed659ccb58744c29bb is 50, key is test_row_0/A:col10/1732536256950/Put/seqid=0 2024-11-25T12:04:17,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:17,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536317349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:17,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536317351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:17,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:17,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:17,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536317363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:17,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741989_1165 (size=12595) 2024-11-25T12:04:17,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T12:04:17,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741990_1166 (size=31549) 2024-11-25T12:04:17,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741991_1167 (size=12304) 2024-11-25T12:04:17,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:17,461 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/72608898eb2b4db4981fa0ef4fc6f01b as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/72608898eb2b4db4981fa0ef4fc6f01b 2024-11-25T12:04:17,489 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c7a06c9b1abb4126bb52048be14658ad_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c7a06c9b1abb4126bb52048be14658ad_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:17,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/967ddf509eb945fc96086db61a51f291, store: [table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:17,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:17,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536317481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:17,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:17,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536317481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:17,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:17,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536317487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:17,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/967ddf509eb945fc96086db61a51f291 is 175, key is test_row_0/A:col10/1732536256950/Put/seqid=0 2024-11-25T12:04:17,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:17,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:17,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:17,517 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/B of 14ed09f1e402e2ed659ccb58744c29bb into 72608898eb2b4db4981fa0ef4fc6f01b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:17,517 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:17,517 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/B, priority=13, startTime=1732536257254; duration=0sec 2024-11-25T12:04:17,517 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:17,517 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:B 2024-11-25T12:04:17,517 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:17,526 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36745 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:17,526 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/C is initiating minor compaction (all files) 2024-11-25T12:04:17,526 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/C in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:17,526 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/2564ae70ce394e86a59dec8616bf63b5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/6384c21f767446a8a5f3e5e872bee1c5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/03c6d5d8b31942f98e98c07e23373f0a] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=35.9 K 2024-11-25T12:04:17,533 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 2564ae70ce394e86a59dec8616bf63b5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732536252357 2024-11-25T12:04:17,537 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 6384c21f767446a8a5f3e5e872bee1c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732536255130 2024-11-25T12:04:17,541 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 03c6d5d8b31942f98e98c07e23373f0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732536256369 2024-11-25T12:04:17,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741992_1168 (size=31105) 2024-11-25T12:04:17,554 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/967ddf509eb945fc96086db61a51f291 2024-11-25T12:04:17,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:17,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536317556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:17,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:17,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/ba7d36e6d48149da87aeee1c70c9f6a8 is 50, key is test_row_0/B:col10/1732536256950/Put/seqid=0 2024-11-25T12:04:17,616 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#C#compaction#139 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:17,618 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/b21bfbedd2b44ebc930fb75140c297e8 is 50, key is test_row_0/C:col10/1732536256730/Put/seqid=0 2024-11-25T12:04:17,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741993_1169 (size=12151) 2024-11-25T12:04:17,657 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/ba7d36e6d48149da87aeee1c70c9f6a8 2024-11-25T12:04:17,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/8ffc50fc96fc43ccbea1411ef3587d24 is 50, key is test_row_0/C:col10/1732536256950/Put/seqid=0 2024-11-25T12:04:17,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741994_1170 (size=12595) 2024-11-25T12:04:17,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536317706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:17,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536317706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:17,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536317721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:17,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741995_1171 (size=12151) 2024-11-25T12:04:17,841 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/3a245cc4528b40aca2d2625b46acd7a9 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/3a245cc4528b40aca2d2625b46acd7a9 2024-11-25T12:04:17,848 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/A of 14ed09f1e402e2ed659ccb58744c29bb into 3a245cc4528b40aca2d2625b46acd7a9(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:17,848 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:17,848 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/A, priority=13, startTime=1732536257253; duration=0sec 2024-11-25T12:04:17,848 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:17,849 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:A 2024-11-25T12:04:17,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T12:04:18,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536318028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:18,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536318028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:18,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536318049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:18,105 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/b21bfbedd2b44ebc930fb75140c297e8 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/b21bfbedd2b44ebc930fb75140c297e8 2024-11-25T12:04:18,135 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/C of 14ed09f1e402e2ed659ccb58744c29bb into b21bfbedd2b44ebc930fb75140c297e8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:18,136 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:18,136 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/C, priority=13, startTime=1732536257254; duration=0sec 2024-11-25T12:04:18,136 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:18,136 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:C 2024-11-25T12:04:18,185 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/8ffc50fc96fc43ccbea1411ef3587d24 2024-11-25T12:04:18,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/967ddf509eb945fc96086db61a51f291 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/967ddf509eb945fc96086db61a51f291 2024-11-25T12:04:18,217 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/967ddf509eb945fc96086db61a51f291, entries=150, sequenceid=209, filesize=30.4 K 2024-11-25T12:04:18,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/ba7d36e6d48149da87aeee1c70c9f6a8 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/ba7d36e6d48149da87aeee1c70c9f6a8 2024-11-25T12:04:18,229 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/ba7d36e6d48149da87aeee1c70c9f6a8, entries=150, sequenceid=209, filesize=11.9 K 2024-11-25T12:04:18,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/8ffc50fc96fc43ccbea1411ef3587d24 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/8ffc50fc96fc43ccbea1411ef3587d24 2024-11-25T12:04:18,238 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/8ffc50fc96fc43ccbea1411ef3587d24, entries=150, sequenceid=209, filesize=11.9 K 2024-11-25T12:04:18,245 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 14ed09f1e402e2ed659ccb58744c29bb in 943ms, sequenceid=209, compaction requested=false 2024-11-25T12:04:18,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:18,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:18,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-25T12:04:18,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-25T12:04:18,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-25T12:04:18,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4460 sec 2024-11-25T12:04:18,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4580 sec 2024-11-25T12:04:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:18,560 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:18,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ed09f1e402e2ed659ccb58744c29bb 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-25T12:04:18,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e1c9152269a44b558f1cb1e3558a0bd7_14ed09f1e402e2ed659ccb58744c29bb is 50, key is test_row_0/A:col10/1732536258555/Put/seqid=0 2024-11-25T12:04:18,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741996_1172 (size=14794) 2024-11-25T12:04:18,643 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:18,650 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125e1c9152269a44b558f1cb1e3558a0bd7_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e1c9152269a44b558f1cb1e3558a0bd7_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:18,652 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/2a4febebfd0e481899818be2fab9882f, store: [table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:18,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/2a4febebfd0e481899818be2fab9882f is 175, key is test_row_0/A:col10/1732536258555/Put/seqid=0 2024-11-25T12:04:18,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741997_1173 (size=39749) 2024-11-25T12:04:18,772 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/2a4febebfd0e481899818be2fab9882f 2024-11-25T12:04:18,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/d4bcff66cb064223bbefa5054a58b14b is 50, key is test_row_0/B:col10/1732536258555/Put/seqid=0 2024-11-25T12:04:18,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536318653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:18,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:18,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536318806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:18,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536318828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:18,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536318830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:18,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741998_1174 (size=12151) 2024-11-25T12:04:18,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/d4bcff66cb064223bbefa5054a58b14b 2024-11-25T12:04:18,860 INFO [master/2bf237fd86ca:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-25T12:04:18,860 INFO [master/2bf237fd86ca:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-25T12:04:18,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/613e60bae36743918e5d6d162efd1f2b is 50, key is test_row_0/C:col10/1732536258555/Put/seqid=0 2024-11-25T12:04:18,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536318917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:18,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741999_1175 (size=12151) 2024-11-25T12:04:18,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/613e60bae36743918e5d6d162efd1f2b 2024-11-25T12:04:18,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536318944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:18,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536318948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:18,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536318948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-25T12:04:18,952 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:18,953 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:04:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-25T12:04:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T12:04:18,958 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:04:18,959 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:04:18,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:04:18,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/2a4febebfd0e481899818be2fab9882f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/2a4febebfd0e481899818be2fab9882f 2024-11-25T12:04:18,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/2a4febebfd0e481899818be2fab9882f, entries=200, sequenceid=226, filesize=38.8 K 2024-11-25T12:04:18,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/d4bcff66cb064223bbefa5054a58b14b as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/d4bcff66cb064223bbefa5054a58b14b 2024-11-25T12:04:19,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/d4bcff66cb064223bbefa5054a58b14b, entries=150, sequenceid=226, filesize=11.9 K 2024-11-25T12:04:19,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/613e60bae36743918e5d6d162efd1f2b as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/613e60bae36743918e5d6d162efd1f2b 2024-11-25T12:04:19,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/613e60bae36743918e5d6d162efd1f2b, entries=150, sequenceid=226, filesize=11.9 K 2024-11-25T12:04:19,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=140.89 KB/144270 for 14ed09f1e402e2ed659ccb58744c29bb in 506ms, sequenceid=226, compaction requested=true 2024-11-25T12:04:19,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:19,068 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:19,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:04:19,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:19,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:04:19,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:19,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:04:19,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:04:19,072 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:19,076 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:19,076 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/A is initiating minor compaction (all files) 2024-11-25T12:04:19,076 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/A in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:19,076 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/3a245cc4528b40aca2d2625b46acd7a9, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/967ddf509eb945fc96086db61a51f291, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/2a4febebfd0e481899818be2fab9882f] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=100.0 K 2024-11-25T12:04:19,076 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:19,076 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/3a245cc4528b40aca2d2625b46acd7a9, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/967ddf509eb945fc96086db61a51f291, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/2a4febebfd0e481899818be2fab9882f] 2024-11-25T12:04:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T12:04:19,078 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a245cc4528b40aca2d2625b46acd7a9, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732536256369 2024-11-25T12:04:19,079 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:19,079 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/B is initiating minor compaction (all files) 2024-11-25T12:04:19,079 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/B in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:19,079 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/72608898eb2b4db4981fa0ef4fc6f01b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/ba7d36e6d48149da87aeee1c70c9f6a8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/d4bcff66cb064223bbefa5054a58b14b] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=36.0 K 2024-11-25T12:04:19,079 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 967ddf509eb945fc96086db61a51f291, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732536256859 2024-11-25T12:04:19,080 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 72608898eb2b4db4981fa0ef4fc6f01b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732536256369 2024-11-25T12:04:19,080 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a4febebfd0e481899818be2fab9882f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732536257337 2024-11-25T12:04:19,081 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting ba7d36e6d48149da87aeee1c70c9f6a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732536256859 2024-11-25T12:04:19,082 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting d4bcff66cb064223bbefa5054a58b14b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732536257337 2024-11-25T12:04:19,105 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:19,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-25T12:04:19,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:19,114 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 14ed09f1e402e2ed659ccb58744c29bb 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T12:04:19,118 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#B#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:19,119 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/c691d6ebc56644229a39f4f56e2aac1d is 50, key is test_row_0/B:col10/1732536258555/Put/seqid=0 2024-11-25T12:04:19,130 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411252cec11ce6f02415c989f0c6c57d6148c_14ed09f1e402e2ed659ccb58744c29bb store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:19,132 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411252cec11ce6f02415c989f0c6c57d6148c_14ed09f1e402e2ed659ccb58744c29bb, store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:19,132 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252cec11ce6f02415c989f0c6c57d6148c_14ed09f1e402e2ed659ccb58744c29bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:19,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:19,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253d1da01a3d694fee81a7d4022063c509_14ed09f1e402e2ed659ccb58744c29bb is 50, key is test_row_0/A:col10/1732536258658/Put/seqid=0 2024-11-25T12:04:19,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536319186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:19,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:19,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536319188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:19,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536319189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:19,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536319191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:19,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742000_1176 (size=12697) 2024-11-25T12:04:19,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742002_1178 (size=12304) 2024-11-25T12:04:19,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742001_1177 (size=4469) 2024-11-25T12:04:19,261 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#A#compaction#144 average throughput is 0.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:19,262 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/7fdfdf72822a49679ad9342531e49ff2 is 175, key is test_row_0/A:col10/1732536258555/Put/seqid=0 2024-11-25T12:04:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T12:04:19,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742003_1179 (size=31651) 2024-11-25T12:04:19,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:19,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536319310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536319311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:19,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536319311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:19,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:19,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536319313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:19,363 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/7fdfdf72822a49679ad9342531e49ff2 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/7fdfdf72822a49679ad9342531e49ff2 2024-11-25T12:04:19,419 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/A of 14ed09f1e402e2ed659ccb58744c29bb into 7fdfdf72822a49679ad9342531e49ff2(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:19,420 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:19,420 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/A, priority=13, startTime=1732536259068; duration=0sec 2024-11-25T12:04:19,420 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:19,420 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:A 2024-11-25T12:04:19,420 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:19,437 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:19,437 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/C is initiating minor compaction (all files) 2024-11-25T12:04:19,437 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/C in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:19,437 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/b21bfbedd2b44ebc930fb75140c297e8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/8ffc50fc96fc43ccbea1411ef3587d24, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/613e60bae36743918e5d6d162efd1f2b] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=36.0 K 2024-11-25T12:04:19,444 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting b21bfbedd2b44ebc930fb75140c297e8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732536256369 2024-11-25T12:04:19,449 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ffc50fc96fc43ccbea1411ef3587d24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732536256859 2024-11-25T12:04:19,457 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 613e60bae36743918e5d6d162efd1f2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732536257337 2024-11-25T12:04:19,507 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#C#compaction#147 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:19,507 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/bfa23d210bb44d1cbe25ae537a8e7753 is 50, key is test_row_0/C:col10/1732536258555/Put/seqid=0 2024-11-25T12:04:19,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:19,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536319527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:19,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536319527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:19,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:19,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536319527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536319527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:19,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:19,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742004_1180 (size=12697) 2024-11-25T12:04:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T12:04:19,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:19,669 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253d1da01a3d694fee81a7d4022063c509_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253d1da01a3d694fee81a7d4022063c509_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:19,679 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/c691d6ebc56644229a39f4f56e2aac1d as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/c691d6ebc56644229a39f4f56e2aac1d 2024-11-25T12:04:19,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/75ca074756aa40608b601b4451985af4, store: [table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:19,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/75ca074756aa40608b601b4451985af4 is 175, key is test_row_0/A:col10/1732536258658/Put/seqid=0 2024-11-25T12:04:19,717 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/B of 14ed09f1e402e2ed659ccb58744c29bb into c691d6ebc56644229a39f4f56e2aac1d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:19,717 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:19,717 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/B, priority=13, startTime=1732536259068; duration=0sec 2024-11-25T12:04:19,718 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:19,718 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:B 2024-11-25T12:04:19,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742005_1181 (size=31105) 2024-11-25T12:04:19,728 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/75ca074756aa40608b601b4451985af4 2024-11-25T12:04:19,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/758a9e2d6d764c72a836f7cc923e593e is 50, key is test_row_0/B:col10/1732536258658/Put/seqid=0 2024-11-25T12:04:19,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742006_1182 (size=12151) 2024-11-25T12:04:19,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536319848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:19,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:19,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:19,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536319849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:19,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:19,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536319849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:19,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:19,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:19,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:19,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536319850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:19,983 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/bfa23d210bb44d1cbe25ae537a8e7753 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/bfa23d210bb44d1cbe25ae537a8e7753 2024-11-25T12:04:20,005 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/C of 14ed09f1e402e2ed659ccb58744c29bb into bfa23d210bb44d1cbe25ae537a8e7753(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:20,005 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:20,005 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/C, priority=13, startTime=1732536259068; duration=0sec 2024-11-25T12:04:20,006 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:20,006 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:C 2024-11-25T12:04:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T12:04:20,205 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/758a9e2d6d764c72a836f7cc923e593e 2024-11-25T12:04:20,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/786919e3925f4796a7324305ba642c49 is 50, key is test_row_0/C:col10/1732536258658/Put/seqid=0 2024-11-25T12:04:20,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742007_1183 (size=12151) 2024-11-25T12:04:20,282 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/786919e3925f4796a7324305ba642c49 2024-11-25T12:04:20,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/75ca074756aa40608b601b4451985af4 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/75ca074756aa40608b601b4451985af4 2024-11-25T12:04:20,352 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/75ca074756aa40608b601b4451985af4, entries=150, sequenceid=250, filesize=30.4 K 2024-11-25T12:04:20,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/758a9e2d6d764c72a836f7cc923e593e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/758a9e2d6d764c72a836f7cc923e593e 2024-11-25T12:04:20,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,378 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/758a9e2d6d764c72a836f7cc923e593e, entries=150, sequenceid=250, filesize=11.9 K 2024-11-25T12:04:20,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/786919e3925f4796a7324305ba642c49 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/786919e3925f4796a7324305ba642c49 2024-11-25T12:04:20,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536320368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536320368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:20,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536320369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:20,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:20,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:20,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:20,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536320369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:20,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,405 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/786919e3925f4796a7324305ba642c49, entries=150, sequenceid=250, filesize=11.9 K 2024-11-25T12:04:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,407 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 14ed09f1e402e2ed659ccb58744c29bb in 1292ms, sequenceid=250, compaction requested=false 2024-11-25T12:04:20,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:20,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:20,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-25T12:04:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-25T12:04:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,418 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-25T12:04:20,418 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4530 sec 2024-11-25T12:04:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,422 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.4670 sec 2024-11-25T12:04:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:20,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-25T12:04:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,131 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:21,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,132 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:04:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-25T12:04:21,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T12:04:21,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,147 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:04:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,157 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:04:21,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:04:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T12:04:21,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-25T12:04:21,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:21,324 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 14ed09f1e402e2ed659ccb58744c29bb 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T12:04:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250149b3139029462fa5a6b41169f432da_14ed09f1e402e2ed659ccb58744c29bb is 50, key is test_row_0/A:col10/1732536259189/Put/seqid=0 2024-11-25T12:04:21,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742008_1184 (size=9914) 2024-11-25T12:04:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:21,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,422 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250149b3139029462fa5a6b41169f432da_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250149b3139029462fa5a6b41169f432da_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/9f8adedeb4734b6da40ad6ab33e9654e, store: [table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/9f8adedeb4734b6da40ad6ab33e9654e is 175, key is test_row_0/A:col10/1732536259189/Put/seqid=0 2024-11-25T12:04:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T12:04:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:21,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742009_1185 (size=22561) 2024-11-25T12:04:21,497 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/9f8adedeb4734b6da40ad6ab33e9654e 2024-11-25T12:04:21,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/67528af7244742869587d2651f76475c is 50, key is test_row_0/B:col10/1732536259189/Put/seqid=0 2024-11-25T12:04:21,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536321546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:21,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536321549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:21,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536321559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:21,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536321568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:21,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742010_1186 (size=9857) 2024-11-25T12:04:21,628 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/67528af7244742869587d2651f76475c 2024-11-25T12:04:21,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536321676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:21,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536321676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:21,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:21,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536321687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:21,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/8324ca97fefc42eb897aafba88cc6c50 is 50, key is test_row_0/C:col10/1732536259189/Put/seqid=0 2024-11-25T12:04:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:21,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536321689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:21,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742011_1187 (size=9857) 2024-11-25T12:04:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T12:04:21,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:21,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536321887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:21,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:21,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536321900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:21,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536321900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:21,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:21,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536321916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:21,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:21,918 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:22,161 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/8324ca97fefc42eb897aafba88cc6c50 2024-11-25T12:04:22,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/9f8adedeb4734b6da40ad6ab33e9654e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/9f8adedeb4734b6da40ad6ab33e9654e 2024-11-25T12:04:22,207 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/9f8adedeb4734b6da40ad6ab33e9654e, entries=100, sequenceid=265, filesize=22.0 K 2024-11-25T12:04:22,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/67528af7244742869587d2651f76475c as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/67528af7244742869587d2651f76475c 2024-11-25T12:04:22,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536322208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:22,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536322216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:22,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536322218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:22,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:22,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:22,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:22,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536322229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:22,239 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/67528af7244742869587d2651f76475c, entries=100, sequenceid=265, filesize=9.6 K 2024-11-25T12:04:22,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/8324ca97fefc42eb897aafba88cc6c50 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/8324ca97fefc42eb897aafba88cc6c50 2024-11-25T12:04:22,266 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/8324ca97fefc42eb897aafba88cc6c50, entries=100, sequenceid=265, filesize=9.6 K 2024-11-25T12:04:22,267 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 14ed09f1e402e2ed659ccb58744c29bb in 943ms, sequenceid=265, compaction requested=true 2024-11-25T12:04:22,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:22,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:22,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-25T12:04:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-25T12:04:22,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-25T12:04:22,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1160 sec 2024-11-25T12:04:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T12:04:22,288 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.1550 sec 2024-11-25T12:04:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:22,756 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:22,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ed09f1e402e2ed659ccb58744c29bb 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-25T12:04:22,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536322778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:22,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536322788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:22,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536322788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:22,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112584c97371f8c342d682627a8a995d9b44_14ed09f1e402e2ed659ccb58744c29bb is 50, key is test_row_0/A:col10/1732536262749/Put/seqid=0 2024-11-25T12:04:22,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536322794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:22,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742012_1188 (size=14994) 2024-11-25T12:04:22,903 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:22,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536322900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:22,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:22,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536322906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:22,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,916 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:22,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536322917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:22,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:22,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536322926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:22,951 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112584c97371f8c342d682627a8a995d9b44_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112584c97371f8c342d682627a8a995d9b44_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:22,953 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/241e4aab1b0144628d21dc1d09f29d52, store: [table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:22,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/241e4aab1b0144628d21dc1d09f29d52 is 175, key is test_row_0/A:col10/1732536262749/Put/seqid=0 2024-11-25T12:04:22,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742013_1189 (size=39949) 2024-11-25T12:04:23,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536323118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:23,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536323126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:23,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:23,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536323139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:23,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536323141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-25T12:04:23,297 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:23,301 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:04:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-25T12:04:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T12:04:23,318 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:04:23,319 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:04:23,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:04:23,398 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/241e4aab1b0144628d21dc1d09f29d52 2024-11-25T12:04:23,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/6e9ddb810d53410a817cb843d8b09905 is 50, key is test_row_0/B:col10/1732536262749/Put/seqid=0 2024-11-25T12:04:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T12:04:23,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536323441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:23,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536323462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:23,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:23,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:23,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536323457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:23,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742014_1190 (size=12301) 2024-11-25T12:04:23,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T12:04:23,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:23,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:23,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:23,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:23,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:23,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536323466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:23,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:23,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/6e9ddb810d53410a817cb843d8b09905 2024-11-25T12:04:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:23,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/aa58d95741704e2cbebc7ead7b9e7f24 is 50, key is test_row_0/C:col10/1732536262749/Put/seqid=0 2024-11-25T12:04:23,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742015_1191 (size=12301) 2024-11-25T12:04:23,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T12:04:23,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:23,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:23,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:23,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:23,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:23,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T12:04:23,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T12:04:23,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:23,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:23,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:23,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:23,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:23,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T12:04:23,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T12:04:23,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:23,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:23,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:23,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:23,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:23,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536323957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:23,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536323978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536323978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:23,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/aa58d95741704e2cbebc7ead7b9e7f24 2024-11-25T12:04:23,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536323988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:24,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/241e4aab1b0144628d21dc1d09f29d52 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/241e4aab1b0144628d21dc1d09f29d52 2024-11-25T12:04:24,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/241e4aab1b0144628d21dc1d09f29d52, entries=200, sequenceid=292, filesize=39.0 K 2024-11-25T12:04:24,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/6e9ddb810d53410a817cb843d8b09905 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/6e9ddb810d53410a817cb843d8b09905 2024-11-25T12:04:24,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T12:04:24,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:24,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:24,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:24,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:24,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:24,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:24,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/6e9ddb810d53410a817cb843d8b09905, entries=150, sequenceid=292, filesize=12.0 K 2024-11-25T12:04:24,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/aa58d95741704e2cbebc7ead7b9e7f24 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/aa58d95741704e2cbebc7ead7b9e7f24 2024-11-25T12:04:24,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/aa58d95741704e2cbebc7ead7b9e7f24, entries=150, sequenceid=292, filesize=12.0 K 2024-11-25T12:04:24,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 14ed09f1e402e2ed659ccb58744c29bb in 1441ms, sequenceid=292, compaction requested=true 2024-11-25T12:04:24,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:24,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:04:24,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:24,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:04:24,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:04:24,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:04:24,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:04:24,197 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:04:24,200 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:04:24,211 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:04:24,211 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/A is initiating minor compaction (all files) 2024-11-25T12:04:24,211 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/A in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:24,212 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/7fdfdf72822a49679ad9342531e49ff2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/75ca074756aa40608b601b4451985af4, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/9f8adedeb4734b6da40ad6ab33e9654e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/241e4aab1b0144628d21dc1d09f29d52] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=122.3 K 2024-11-25T12:04:24,212 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:24,212 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/7fdfdf72822a49679ad9342531e49ff2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/75ca074756aa40608b601b4451985af4, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/9f8adedeb4734b6da40ad6ab33e9654e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/241e4aab1b0144628d21dc1d09f29d52] 2024-11-25T12:04:24,217 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47006 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:04:24,217 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/C is initiating minor compaction (all files) 2024-11-25T12:04:24,217 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/C in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:24,217 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/bfa23d210bb44d1cbe25ae537a8e7753, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/786919e3925f4796a7324305ba642c49, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/8324ca97fefc42eb897aafba88cc6c50, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/aa58d95741704e2cbebc7ead7b9e7f24] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=45.9 K 2024-11-25T12:04:24,217 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fdfdf72822a49679ad9342531e49ff2, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732536257337 2024-11-25T12:04:24,218 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting bfa23d210bb44d1cbe25ae537a8e7753, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732536257337 2024-11-25T12:04:24,218 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75ca074756aa40608b601b4451985af4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732536258642 2024-11-25T12:04:24,225 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 786919e3925f4796a7324305ba642c49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732536258642 2024-11-25T12:04:24,225 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f8adedeb4734b6da40ad6ab33e9654e, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732536259175 2024-11-25T12:04:24,226 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 241e4aab1b0144628d21dc1d09f29d52, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732536261545 2024-11-25T12:04:24,226 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 8324ca97fefc42eb897aafba88cc6c50, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732536259175 2024-11-25T12:04:24,227 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting aa58d95741704e2cbebc7ead7b9e7f24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732536261558 2024-11-25T12:04:24,246 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#C#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:24,247 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/1e7928509a704d45bb92001cd46e4d6f is 50, key is test_row_0/C:col10/1732536262749/Put/seqid=0 2024-11-25T12:04:24,251 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:24,261 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125ce5d0be5c9e7439bbeb3e3054d3f19b0_14ed09f1e402e2ed659ccb58744c29bb store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:24,263 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125ce5d0be5c9e7439bbeb3e3054d3f19b0_14ed09f1e402e2ed659ccb58744c29bb, store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:24,264 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ce5d0be5c9e7439bbeb3e3054d3f19b0_14ed09f1e402e2ed659ccb58744c29bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:24,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742016_1192 (size=12983) 2024-11-25T12:04:24,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-25T12:04:24,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:24,293 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 14ed09f1e402e2ed659ccb58744c29bb 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-25T12:04:24,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742017_1193 (size=4469) 2024-11-25T12:04:24,340 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#A#compaction#157 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:24,341 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/db86b847907946f09390c6daf6f99214 is 175, key is test_row_0/A:col10/1732536262749/Put/seqid=0 2024-11-25T12:04:24,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125844d1cf3d4914ba18b1c012a56df0496_14ed09f1e402e2ed659ccb58744c29bb is 50, key is test_row_0/A:col10/1732536262785/Put/seqid=0 2024-11-25T12:04:24,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742018_1194 (size=31937) 2024-11-25T12:04:24,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742019_1195 (size=12454) 2024-11-25T12:04:24,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:24,423 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125844d1cf3d4914ba18b1c012a56df0496_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125844d1cf3d4914ba18b1c012a56df0496_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:24,423 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/db86b847907946f09390c6daf6f99214 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/db86b847907946f09390c6daf6f99214 2024-11-25T12:04:24,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/3ea9ef9e080e41298d7fdba7f85bc6be, store: [table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:24,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/3ea9ef9e080e41298d7fdba7f85bc6be is 175, key is test_row_0/A:col10/1732536262785/Put/seqid=0 2024-11-25T12:04:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T12:04:24,457 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/A of 14ed09f1e402e2ed659ccb58744c29bb into db86b847907946f09390c6daf6f99214(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:24,457 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:24,457 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/A, priority=12, startTime=1732536264197; duration=0sec 2024-11-25T12:04:24,458 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:24,458 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:A 2024-11-25T12:04:24,458 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:04:24,472 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47006 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:04:24,473 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/B is initiating minor compaction (all files) 2024-11-25T12:04:24,473 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/B in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:24,473 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/c691d6ebc56644229a39f4f56e2aac1d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/758a9e2d6d764c72a836f7cc923e593e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/67528af7244742869587d2651f76475c, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/6e9ddb810d53410a817cb843d8b09905] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=45.9 K 2024-11-25T12:04:24,475 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting c691d6ebc56644229a39f4f56e2aac1d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732536257337 2024-11-25T12:04:24,475 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 758a9e2d6d764c72a836f7cc923e593e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1732536258642 2024-11-25T12:04:24,479 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67528af7244742869587d2651f76475c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732536259175 2024-11-25T12:04:24,479 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e9ddb810d53410a817cb843d8b09905, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732536261558 2024-11-25T12:04:24,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742020_1196 (size=31255) 2024-11-25T12:04:24,518 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#B#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:24,518 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/31cc30d02c8a4d4894326c7ed4e3ba2a is 50, key is test_row_0/B:col10/1732536262749/Put/seqid=0 2024-11-25T12:04:24,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742021_1197 (size=12983) 2024-11-25T12:04:24,696 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/1e7928509a704d45bb92001cd46e4d6f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/1e7928509a704d45bb92001cd46e4d6f 2024-11-25T12:04:24,709 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/C of 14ed09f1e402e2ed659ccb58744c29bb into 1e7928509a704d45bb92001cd46e4d6f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:24,709 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:24,709 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/C, priority=12, startTime=1732536264197; duration=0sec 2024-11-25T12:04:24,709 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:24,710 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:C 2024-11-25T12:04:24,901 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=301, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/3ea9ef9e080e41298d7fdba7f85bc6be 2024-11-25T12:04:24,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/b5b02e2bbc6f4d48941c67295fc8ed44 is 50, key is test_row_0/B:col10/1732536262785/Put/seqid=0 2024-11-25T12:04:24,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742022_1198 (size=12301) 2024-11-25T12:04:24,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. as already flushing 2024-11-25T12:04:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:25,053 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/31cc30d02c8a4d4894326c7ed4e3ba2a as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/31cc30d02c8a4d4894326c7ed4e3ba2a 2024-11-25T12:04:25,069 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/B of 14ed09f1e402e2ed659ccb58744c29bb into 31cc30d02c8a4d4894326c7ed4e3ba2a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:25,069 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:25,069 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/B, priority=12, startTime=1732536264197; duration=0sec 2024-11-25T12:04:25,069 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:25,069 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:B 2024-11-25T12:04:25,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:25,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536325118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:25,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:25,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536325119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:25,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536325124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:25,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536325131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:25,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:25,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536325237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:25,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:25,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:25,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536325249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:25,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:25,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:25,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536325250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:25,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:25,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536325254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:25,387 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/b5b02e2bbc6f4d48941c67295fc8ed44 2024-11-25T12:04:25,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/0a56160977da4b768a198a34a2cc0a48 is 50, key is test_row_0/C:col10/1732536262785/Put/seqid=0 2024-11-25T12:04:25,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536325452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:25,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742023_1199 (size=12301) 2024-11-25T12:04:25,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:25,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536325457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:25,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:25,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T12:04:25,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:25,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536325475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:25,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536325476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:25,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:25,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:25,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536325769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:25,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:25,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536325776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:25,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536325787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:25,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:25,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536325787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:25,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:25,864 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/0a56160977da4b768a198a34a2cc0a48 2024-11-25T12:04:25,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:25,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60736 deadline: 1732536325866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:25,874 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_2' in 14ed09f1e402e2ed659ccb58744c29bb of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18404 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:04:25,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:25,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:25,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/3ea9ef9e080e41298d7fdba7f85bc6be as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/3ea9ef9e080e41298d7fdba7f85bc6be 2024-11-25T12:04:25,913 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/3ea9ef9e080e41298d7fdba7f85bc6be, entries=150, sequenceid=301, filesize=30.5 K 2024-11-25T12:04:25,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/b5b02e2bbc6f4d48941c67295fc8ed44 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/b5b02e2bbc6f4d48941c67295fc8ed44 2024-11-25T12:04:25,958 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/b5b02e2bbc6f4d48941c67295fc8ed44, entries=150, sequenceid=301, filesize=12.0 K 2024-11-25T12:04:25,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/0a56160977da4b768a198a34a2cc0a48 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/0a56160977da4b768a198a34a2cc0a48 2024-11-25T12:04:25,988 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/0a56160977da4b768a198a34a2cc0a48, entries=150, sequenceid=301, filesize=12.0 K 2024-11-25T12:04:25,989 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for 14ed09f1e402e2ed659ccb58744c29bb in 1696ms, sequenceid=301, compaction requested=false 2024-11-25T12:04:25,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:25,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:25,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-25T12:04:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-25T12:04:26,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-25T12:04:26,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6820 sec 2024-11-25T12:04:26,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.7070 sec 2024-11-25T12:04:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:26,281 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:26,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ed09f1e402e2ed659ccb58744c29bb 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-25T12:04:26,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536326296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:26,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536326296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:26,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536326299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:26,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536326302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:26,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:26,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:26,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:26,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c6681523879f40eab16ca6af35416a75_14ed09f1e402e2ed659ccb58744c29bb is 50, key is test_row_0/A:col10/1732536265111/Put/seqid=0 2024-11-25T12:04:26,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742024_1200 (size=14994) 2024-11-25T12:04:26,358 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,364 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c6681523879f40eab16ca6af35416a75_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c6681523879f40eab16ca6af35416a75_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:26,366 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/13dee5bb442d487aa0b84ae4d0b66e42, store: [table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:26,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/13dee5bb442d487aa0b84ae4d0b66e42 is 175, key is test_row_0/A:col10/1732536265111/Put/seqid=0 2024-11-25T12:04:26,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742025_1201 (size=39949) 2024-11-25T12:04:26,399 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/13dee5bb442d487aa0b84ae4d0b66e42 2024-11-25T12:04:26,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536326409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:26,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536326410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:26,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/cc2411fee4154d5d8f27fc1150bc56a0 is 50, key is test_row_0/B:col10/1732536265111/Put/seqid=0 2024-11-25T12:04:26,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742026_1202 (size=12301) 2024-11-25T12:04:26,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/cc2411fee4154d5d8f27fc1150bc56a0 2024-11-25T12:04:26,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/5228c4c61e40474192d131d45221e0b0 is 50, key is test_row_0/C:col10/1732536265111/Put/seqid=0 2024-11-25T12:04:26,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742027_1203 (size=12301) 2024-11-25T12:04:26,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/5228c4c61e40474192d131d45221e0b0 2024-11-25T12:04:26,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/13dee5bb442d487aa0b84ae4d0b66e42 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/13dee5bb442d487aa0b84ae4d0b66e42 2024-11-25T12:04:26,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/13dee5bb442d487aa0b84ae4d0b66e42, entries=200, sequenceid=333, filesize=39.0 K 2024-11-25T12:04:26,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/cc2411fee4154d5d8f27fc1150bc56a0 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/cc2411fee4154d5d8f27fc1150bc56a0 2024-11-25T12:04:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/cc2411fee4154d5d8f27fc1150bc56a0, entries=150, sequenceid=333, filesize=12.0 K 2024-11-25T12:04:26,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/5228c4c61e40474192d131d45221e0b0 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/5228c4c61e40474192d131d45221e0b0 2024-11-25T12:04:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/5228c4c61e40474192d131d45221e0b0, entries=150, sequenceid=333, filesize=12.0 K 2024-11-25T12:04:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 14ed09f1e402e2ed659ccb58744c29bb in 343ms, sequenceid=333, compaction requested=true 2024-11-25T12:04:26,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:04:26,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:26,625 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:26,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:04:26,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:26,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:04:26,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:26,625 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,634 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:26,634 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/A is initiating minor compaction (all files) 2024-11-25T12:04:26,634 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/A in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,635 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/db86b847907946f09390c6daf6f99214, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/3ea9ef9e080e41298d7fdba7f85bc6be, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/13dee5bb442d487aa0b84ae4d0b66e42] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=100.7 K 2024-11-25T12:04:26,635 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:26,635 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/db86b847907946f09390c6daf6f99214, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/3ea9ef9e080e41298d7fdba7f85bc6be, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/13dee5bb442d487aa0b84ae4d0b66e42] 2024-11-25T12:04:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,636 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting db86b847907946f09390c6daf6f99214, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732536261558 2024-11-25T12:04:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,636 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ea9ef9e080e41298d7fdba7f85bc6be, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732536262770 2024-11-25T12:04:26,636 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13dee5bb442d487aa0b84ae4d0b66e42, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732536265111 2024-11-25T12:04:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,644 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:26,644 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/B is initiating minor compaction (all files) 2024-11-25T12:04:26,644 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/B in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,645 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:26,645 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/31cc30d02c8a4d4894326c7ed4e3ba2a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/b5b02e2bbc6f4d48941c67295fc8ed44, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/cc2411fee4154d5d8f27fc1150bc56a0] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=36.7 K 2024-11-25T12:04:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,646 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 31cc30d02c8a4d4894326c7ed4e3ba2a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732536261558 2024-11-25T12:04:26,647 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting b5b02e2bbc6f4d48941c67295fc8ed44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732536262770 2024-11-25T12:04:26,647 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting cc2411fee4154d5d8f27fc1150bc56a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732536265111 2024-11-25T12:04:26,648 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125146f3e1e96a8450bad67b5805d102a7d_14ed09f1e402e2ed659ccb58744c29bb store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,651 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125146f3e1e96a8450bad67b5805d102a7d_14ed09f1e402e2ed659ccb58744c29bb, store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:26,651 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125146f3e1e96a8450bad67b5805d102a7d_14ed09f1e402e2ed659ccb58744c29bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,677 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#B#compaction#166 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:26,678 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/18ff5030548546849c24b5b502139c82 is 50, key is test_row_0/B:col10/1732536265111/Put/seqid=0 2024-11-25T12:04:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742028_1204 (size=4469) 2024-11-25T12:04:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742029_1205 (size=13085) 2024-11-25T12:04:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,714 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:26,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ed09f1e402e2ed659ccb58744c29bb 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:04:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,721 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/18ff5030548546849c24b5b502139c82 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/18ff5030548546849c24b5b502139c82 2024-11-25T12:04:26,729 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/B of 14ed09f1e402e2ed659ccb58744c29bb into 18ff5030548546849c24b5b502139c82(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:26,729 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:26,729 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/B, priority=13, startTime=1732536266625; duration=0sec 2024-11-25T12:04:26,729 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:26,729 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:B 2024-11-25T12:04:26,729 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:26,733 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:26,734 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/C is initiating minor compaction (all files) 2024-11-25T12:04:26,734 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/C in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:26,734 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/1e7928509a704d45bb92001cd46e4d6f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/0a56160977da4b768a198a34a2cc0a48, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/5228c4c61e40474192d131d45221e0b0] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=36.7 K 2024-11-25T12:04:26,742 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e7928509a704d45bb92001cd46e4d6f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732536261558 2024-11-25T12:04:26,743 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a56160977da4b768a198a34a2cc0a48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732536262770 2024-11-25T12:04:26,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112594b8c020074f435192869b6b58f4f4f0_14ed09f1e402e2ed659ccb58744c29bb is 50, key is test_row_0/A:col10/1732536266691/Put/seqid=0 2024-11-25T12:04:26,747 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 5228c4c61e40474192d131d45221e0b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732536265111 2024-11-25T12:04:26,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742030_1206 (size=20074) 2024-11-25T12:04:26,766 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:26,775 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#C#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:26,776 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/a603ee27a00e4de9b149e027c0caa492 is 50, key is test_row_0/C:col10/1732536265111/Put/seqid=0 2024-11-25T12:04:26,785 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112594b8c020074f435192869b6b58f4f4f0_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112594b8c020074f435192869b6b58f4f4f0_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:26,788 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/d2a6b7fe6de444f1b478805e1e6b4812, store: [table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:26,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/d2a6b7fe6de444f1b478805e1e6b4812 is 175, key is test_row_0/A:col10/1732536266691/Put/seqid=0 2024-11-25T12:04:26,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742031_1207 (size=13085) 2024-11-25T12:04:26,832 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/a603ee27a00e4de9b149e027c0caa492 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/a603ee27a00e4de9b149e027c0caa492 2024-11-25T12:04:26,841 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/C of 14ed09f1e402e2ed659ccb58744c29bb into a603ee27a00e4de9b149e027c0caa492(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:26,841 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:26,841 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/C, priority=13, startTime=1732536266625; duration=0sec 2024-11-25T12:04:26,841 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:26,842 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:C 2024-11-25T12:04:26,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742032_1208 (size=57333) 2024-11-25T12:04:26,848 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/d2a6b7fe6de444f1b478805e1e6b4812 2024-11-25T12:04:26,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/fc679f1e713e46e99b0c6637ef40ca81 is 50, key is test_row_0/B:col10/1732536266691/Put/seqid=0 2024-11-25T12:04:26,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:26,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:26,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536326869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:26,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536326873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:26,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742033_1209 (size=12301) 2024-11-25T12:04:26,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/fc679f1e713e46e99b0c6637ef40ca81 2024-11-25T12:04:26,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/2709da7fe27943a68621e8c5d32108f6 is 50, key is test_row_0/C:col10/1732536266691/Put/seqid=0 2024-11-25T12:04:26,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742034_1210 (size=12301) 2024-11-25T12:04:26,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536326989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:26,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536326989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:27,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:27,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:27,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:27,093 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#A#compaction#165 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:27,094 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/1eec1b1fb1d049b4a1a7521ce9e90646 is 175, key is test_row_0/A:col10/1732536265111/Put/seqid=0 2024-11-25T12:04:27,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742035_1211 (size=32039) 2024-11-25T12:04:27,157 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/1eec1b1fb1d049b4a1a7521ce9e90646 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/1eec1b1fb1d049b4a1a7521ce9e90646 2024-11-25T12:04:27,197 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/A of 14ed09f1e402e2ed659ccb58744c29bb into 1eec1b1fb1d049b4a1a7521ce9e90646(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:27,197 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:27,197 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/A, priority=13, startTime=1732536266625; duration=0sec 2024-11-25T12:04:27,197 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:27,197 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:A 2024-11-25T12:04:27,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:27,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60742 deadline: 1732536327208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60754 deadline: 1732536327208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:27,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:27,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:27,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:27,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:27,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:27,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:27,305 INFO [Thread-587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-587. 2024-11-25T12:04:27,305 DEBUG [Thread-587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:04:27,305 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:27,305 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:27,305 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:04:27,309 INFO [Thread-585 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-585. 2024-11-25T12:04:27,309 DEBUG [Thread-585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:04:27,309 DEBUG [Thread-585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:27,309 DEBUG [Thread-585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:27,309 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:04:27,314 INFO [Thread-589 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-589. 2024-11-25T12:04:27,314 DEBUG [Thread-589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:04:27,314 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:27,314 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:27,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:04:27,316 INFO [Thread-591 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-591. 2024-11-25T12:04:27,316 DEBUG [Thread-591 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:04:27,316 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:27,316 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:27,316 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:04:27,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60758 deadline: 1732536327317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:27,319 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_1' in 14ed09f1e402e2ed659ccb58744c29bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:04:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:27,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60714 deadline: 1732536327318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:27,321 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_2' in 14ed09f1e402e2ed659ccb58744c29bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:04:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:04:27,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/2709da7fe27943a68621e8c5d32108f6 2024-11-25T12:04:27,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/d2a6b7fe6de444f1b478805e1e6b4812 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/d2a6b7fe6de444f1b478805e1e6b4812 2024-11-25T12:04:27,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/d2a6b7fe6de444f1b478805e1e6b4812, entries=300, sequenceid=344, filesize=56.0 K 2024-11-25T12:04:27,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/fc679f1e713e46e99b0c6637ef40ca81 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/fc679f1e713e46e99b0c6637ef40ca81 2024-11-25T12:04:27,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/fc679f1e713e46e99b0c6637ef40ca81, entries=150, sequenceid=344, filesize=12.0 K 2024-11-25T12:04:27,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/2709da7fe27943a68621e8c5d32108f6 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/2709da7fe27943a68621e8c5d32108f6 2024-11-25T12:04:27,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/2709da7fe27943a68621e8c5d32108f6, entries=150, sequenceid=344, filesize=12.0 K 2024-11-25T12:04:27,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 14ed09f1e402e2ed659ccb58744c29bb in 699ms, sequenceid=344, compaction requested=false 2024-11-25T12:04:27,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-25T12:04:27,489 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:27,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:27,529 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:27,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ed09f1e402e2ed659ccb58744c29bb 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-25T12:04:27,530 INFO [Thread-578 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-578. 2024-11-25T12:04:27,530 DEBUG [Thread-578 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:04:27,530 DEBUG [Thread-578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:27,530 DEBUG [Thread-578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:27,530 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:04:27,531 INFO [Thread-582 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-582. 2024-11-25T12:04:27,531 DEBUG [Thread-582 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:04:27,531 DEBUG [Thread-582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:27,531 DEBUG [Thread-582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:27,531 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:04:27,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ec1d8eff9bb64faf8af1ea620ef37ccf_14ed09f1e402e2ed659ccb58744c29bb is 50, key is test_row_0/A:col10/1732536267527/Put/seqid=0 2024-11-25T12:04:27,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742036_1212 (size=12454) 2024-11-25T12:04:27,969 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:27,974 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ec1d8eff9bb64faf8af1ea620ef37ccf_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ec1d8eff9bb64faf8af1ea620ef37ccf_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:27,975 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/58268b9073704bb69ffa4c9213fac5e3, store: [table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:27,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/58268b9073704bb69ffa4c9213fac5e3 is 175, key is test_row_0/A:col10/1732536267527/Put/seqid=0 2024-11-25T12:04:27,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742037_1213 (size=31255) 2024-11-25T12:04:28,385 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/58268b9073704bb69ffa4c9213fac5e3 2024-11-25T12:04:28,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/ffb00da240074c4e91be5c795ba62d40 is 50, key is test_row_0/B:col10/1732536267527/Put/seqid=0 2024-11-25T12:04:28,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742038_1214 (size=12301) 2024-11-25T12:04:28,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/ffb00da240074c4e91be5c795ba62d40 2024-11-25T12:04:28,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/1cc6febb1b934f639249454477731d5b is 50, key is test_row_0/C:col10/1732536267527/Put/seqid=0 2024-11-25T12:04:28,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742039_1215 (size=12301) 2024-11-25T12:04:29,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/1cc6febb1b934f639249454477731d5b 2024-11-25T12:04:29,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/58268b9073704bb69ffa4c9213fac5e3 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/58268b9073704bb69ffa4c9213fac5e3 2024-11-25T12:04:29,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/58268b9073704bb69ffa4c9213fac5e3, entries=150, sequenceid=375, filesize=30.5 K 2024-11-25T12:04:29,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/ffb00da240074c4e91be5c795ba62d40 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/ffb00da240074c4e91be5c795ba62d40 2024-11-25T12:04:29,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/ffb00da240074c4e91be5c795ba62d40, entries=150, sequenceid=375, filesize=12.0 K 2024-11-25T12:04:29,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/1cc6febb1b934f639249454477731d5b as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/1cc6febb1b934f639249454477731d5b 2024-11-25T12:04:29,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/1cc6febb1b934f639249454477731d5b, entries=150, sequenceid=375, filesize=12.0 K 2024-11-25T12:04:29,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=0 B/0 for 14ed09f1e402e2ed659ccb58744c29bb in 1752ms, sequenceid=375, compaction requested=true 2024-11-25T12:04:29,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:29,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:04:29,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:29,281 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:29,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:04:29,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:29,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ed09f1e402e2ed659ccb58744c29bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:04:29,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:29,281 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:29,288 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:29,288 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/A is initiating minor compaction (all files) 2024-11-25T12:04:29,288 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/A in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:29,289 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/1eec1b1fb1d049b4a1a7521ce9e90646, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/d2a6b7fe6de444f1b478805e1e6b4812, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/58268b9073704bb69ffa4c9213fac5e3] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=117.8 K 2024-11-25T12:04:29,289 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:29,289 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/1eec1b1fb1d049b4a1a7521ce9e90646, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/d2a6b7fe6de444f1b478805e1e6b4812, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/58268b9073704bb69ffa4c9213fac5e3] 2024-11-25T12:04:29,288 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:29,289 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/B is initiating minor compaction (all files) 2024-11-25T12:04:29,289 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/B in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:29,289 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/18ff5030548546849c24b5b502139c82, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/fc679f1e713e46e99b0c6637ef40ca81, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/ffb00da240074c4e91be5c795ba62d40] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=36.8 K 2024-11-25T12:04:29,293 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1eec1b1fb1d049b4a1a7521ce9e90646, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732536265111 2024-11-25T12:04:29,294 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 18ff5030548546849c24b5b502139c82, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732536265111 2024-11-25T12:04:29,294 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2a6b7fe6de444f1b478805e1e6b4812, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732536266288 2024-11-25T12:04:29,294 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting fc679f1e713e46e99b0c6637ef40ca81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732536266298 2024-11-25T12:04:29,294 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58268b9073704bb69ffa4c9213fac5e3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732536266798 2024-11-25T12:04:29,295 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting ffb00da240074c4e91be5c795ba62d40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732536266798 2024-11-25T12:04:29,306 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#B#compaction#174 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:29,307 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/779c894874b34dc184729c496eec3c88 is 50, key is test_row_0/B:col10/1732536267527/Put/seqid=0 2024-11-25T12:04:29,321 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:29,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742040_1216 (size=13187) 2024-11-25T12:04:29,324 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125f0a86d4d51654d0c86b615b6bfbd8452_14ed09f1e402e2ed659ccb58744c29bb store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:29,333 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/779c894874b34dc184729c496eec3c88 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/779c894874b34dc184729c496eec3c88 2024-11-25T12:04:29,370 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125f0a86d4d51654d0c86b615b6bfbd8452_14ed09f1e402e2ed659ccb58744c29bb, store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:29,370 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f0a86d4d51654d0c86b615b6bfbd8452_14ed09f1e402e2ed659ccb58744c29bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:29,372 INFO [Thread-580 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-580. 2024-11-25T12:04:29,372 DEBUG [Thread-580 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:04:29,372 DEBUG [Thread-580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:29,372 DEBUG [Thread-580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:29,373 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/B of 14ed09f1e402e2ed659ccb58744c29bb into 779c894874b34dc184729c496eec3c88(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:29,373 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:29,373 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/B, priority=13, startTime=1732536269281; duration=0sec 2024-11-25T12:04:29,373 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:29,373 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:B 2024-11-25T12:04:29,373 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:29,373 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:04:29,381 INFO [Thread-574 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-574. 2024-11-25T12:04:29,381 DEBUG [Thread-574 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:04:29,381 DEBUG [Thread-574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:29,381 DEBUG [Thread-574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:29,381 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:04:29,384 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:29,385 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 14ed09f1e402e2ed659ccb58744c29bb/C is initiating minor compaction (all files) 2024-11-25T12:04:29,385 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ed09f1e402e2ed659ccb58744c29bb/C in TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:29,385 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/a603ee27a00e4de9b149e027c0caa492, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/2709da7fe27943a68621e8c5d32108f6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/1cc6febb1b934f639249454477731d5b] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp, totalSize=36.8 K 2024-11-25T12:04:29,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742041_1217 (size=4469) 2024-11-25T12:04:29,392 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting a603ee27a00e4de9b149e027c0caa492, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732536265111 2024-11-25T12:04:29,393 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 2709da7fe27943a68621e8c5d32108f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732536266298 2024-11-25T12:04:29,393 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cc6febb1b934f639249454477731d5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732536266798 2024-11-25T12:04:29,395 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#A#compaction#175 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:29,396 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/2123afef12874ffab8fb7cc5ab87f1fd is 175, key is test_row_0/A:col10/1732536267527/Put/seqid=0 2024-11-25T12:04:29,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742042_1218 (size=32141) 2024-11-25T12:04:29,437 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ed09f1e402e2ed659ccb58744c29bb#C#compaction#176 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:29,438 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/ac5c52ffc2634f59b5925e010cd9138f is 50, key is test_row_0/C:col10/1732536267527/Put/seqid=0 2024-11-25T12:04:29,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742043_1219 (size=13187) 2024-11-25T12:04:29,493 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/ac5c52ffc2634f59b5925e010cd9138f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/ac5c52ffc2634f59b5925e010cd9138f 2024-11-25T12:04:29,512 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/C of 14ed09f1e402e2ed659ccb58744c29bb into ac5c52ffc2634f59b5925e010cd9138f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:29,512 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:29,512 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/C, priority=13, startTime=1732536269281; duration=0sec 2024-11-25T12:04:29,512 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:29,512 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:C 2024-11-25T12:04:29,830 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/2123afef12874ffab8fb7cc5ab87f1fd as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/2123afef12874ffab8fb7cc5ab87f1fd 2024-11-25T12:04:29,836 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ed09f1e402e2ed659ccb58744c29bb/A of 14ed09f1e402e2ed659ccb58744c29bb into 2123afef12874ffab8fb7cc5ab87f1fd(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:29,836 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ed09f1e402e2ed659ccb58744c29bb: 2024-11-25T12:04:29,836 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., storeName=14ed09f1e402e2ed659ccb58744c29bb/A, priority=13, startTime=1732536269281; duration=0sec 2024-11-25T12:04:29,837 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:29,837 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ed09f1e402e2ed659ccb58744c29bb:A 2024-11-25T12:04:32,416 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T12:04:35,948 INFO [Thread-576 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-576. 2024-11-25T12:04:35,948 DEBUG [Thread-576 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:04:35,948 DEBUG [Thread-576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:35,948 DEBUG [Thread-576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:35,948 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:04:35,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T12:04:35,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-25T12:04:35,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-25T12:04:35,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-25T12:04:35,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-25T12:04:35,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-25T12:04:35,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T12:04:35,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1652 2024-11-25T12:04:35,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1636 2024-11-25T12:04:35,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T12:04:35,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 765 2024-11-25T12:04:35,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2288 rows 2024-11-25T12:04:35,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 763 2024-11-25T12:04:35,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2279 rows 2024-11-25T12:04:35,949 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T12:04:35,949 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:04:35,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:35,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:35,949 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:04:35,953 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-25T12:04:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T12:04:35,957 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536275957"}]},"ts":"1732536275957"} 2024-11-25T12:04:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T12:04:35,959 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T12:04:35,959 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T12:04:35,960 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T12:04:35,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ed09f1e402e2ed659ccb58744c29bb, UNASSIGN}] 2024-11-25T12:04:35,963 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ed09f1e402e2ed659ccb58744c29bb, UNASSIGN 2024-11-25T12:04:35,965 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=14ed09f1e402e2ed659ccb58744c29bb, regionState=CLOSING, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:35,967 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ed09f1e402e2ed659ccb58744c29bb, UNASSIGN because future has completed 2024-11-25T12:04:35,967 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T12:04:35,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674}] 2024-11-25T12:04:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T12:04:36,124 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:36,124 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T12:04:36,125 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing 14ed09f1e402e2ed659ccb58744c29bb, disabling compactions & flushes 2024-11-25T12:04:36,125 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:36,125 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:36,125 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. after waiting 0 ms 2024-11-25T12:04:36,125 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:36,125 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing 14ed09f1e402e2ed659ccb58744c29bb 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-25T12:04:36,149 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254304aa3abd914825aa0e3d1d8de423a1_14ed09f1e402e2ed659ccb58744c29bb is 50, key is test_row_1/A:col10/1732536269348/Put/seqid=0 2024-11-25T12:04:36,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742044_1220 (size=9914) 2024-11-25T12:04:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T12:04:36,562 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:36,569 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254304aa3abd914825aa0e3d1d8de423a1_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254304aa3abd914825aa0e3d1d8de423a1_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:36,573 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/1edb00abbd6c419c92b0afcf07054578, store: [table=TestAcidGuarantees family=A region=14ed09f1e402e2ed659ccb58744c29bb] 2024-11-25T12:04:36,575 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/1edb00abbd6c419c92b0afcf07054578 is 175, key is test_row_1/A:col10/1732536269348/Put/seqid=0 2024-11-25T12:04:36,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742045_1221 (size=22561) 2024-11-25T12:04:36,584 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=384, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/1edb00abbd6c419c92b0afcf07054578 2024-11-25T12:04:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T12:04:36,594 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/183bc7b831a94524acf349dfc169c6df is 50, key is test_row_1/B:col10/1732536269348/Put/seqid=0 2024-11-25T12:04:36,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742046_1222 (size=9857) 2024-11-25T12:04:36,607 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/183bc7b831a94524acf349dfc169c6df 2024-11-25T12:04:36,621 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/c20b6259837647f6b79e61a146922468 is 50, key is test_row_1/C:col10/1732536269348/Put/seqid=0 2024-11-25T12:04:36,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742047_1223 (size=9857) 2024-11-25T12:04:37,039 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/c20b6259837647f6b79e61a146922468 2024-11-25T12:04:37,061 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/A/1edb00abbd6c419c92b0afcf07054578 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/1edb00abbd6c419c92b0afcf07054578 2024-11-25T12:04:37,068 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/1edb00abbd6c419c92b0afcf07054578, entries=100, sequenceid=384, filesize=22.0 K 2024-11-25T12:04:37,069 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/B/183bc7b831a94524acf349dfc169c6df as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/183bc7b831a94524acf349dfc169c6df 2024-11-25T12:04:37,074 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/183bc7b831a94524acf349dfc169c6df, entries=100, sequenceid=384, filesize=9.6 K 2024-11-25T12:04:37,075 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/.tmp/C/c20b6259837647f6b79e61a146922468 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/c20b6259837647f6b79e61a146922468 2024-11-25T12:04:37,084 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/c20b6259837647f6b79e61a146922468, entries=100, sequenceid=384, filesize=9.6 K 2024-11-25T12:04:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T12:04:37,097 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 14ed09f1e402e2ed659ccb58744c29bb in 971ms, sequenceid=384, compaction requested=false 2024-11-25T12:04:37,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/31c384b96e3f48a1851a02dbcc6a9e7e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/4829966e3d264b6dafc90d45a35465e8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/8a1d328198ff400fa884667f0cd4917f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/4d92d500a2d04c7faef954fe16e7f63e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/1cfacb0707664d99b07b913ec3350b86, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/bab8f05ab95d46a89d4ef0ea9bdaf140, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/f1e3cf7d7b694e32a462cb441a82fcb4, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/47b9615aa4634ce8ab178e6b50992dc7, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/dc86c4a9b0824213a7c5e61672bef541, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/ce30aa8125004788b348465b47e3f4b3, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/87a4228dade2496e84d3d244279140e4, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/63175b59da7b451fb6098f1fd4e70ea8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/3a245cc4528b40aca2d2625b46acd7a9, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/afbe494045d84de0873f7fb110e05b36, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/967ddf509eb945fc96086db61a51f291, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/2a4febebfd0e481899818be2fab9882f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/7fdfdf72822a49679ad9342531e49ff2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/75ca074756aa40608b601b4451985af4, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/9f8adedeb4734b6da40ad6ab33e9654e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/241e4aab1b0144628d21dc1d09f29d52, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/db86b847907946f09390c6daf6f99214, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/3ea9ef9e080e41298d7fdba7f85bc6be, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/13dee5bb442d487aa0b84ae4d0b66e42, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/1eec1b1fb1d049b4a1a7521ce9e90646, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/d2a6b7fe6de444f1b478805e1e6b4812, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/58268b9073704bb69ffa4c9213fac5e3] to archive 2024-11-25T12:04:37,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:04:37,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/31c384b96e3f48a1851a02dbcc6a9e7e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/31c384b96e3f48a1851a02dbcc6a9e7e 2024-11-25T12:04:37,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/4829966e3d264b6dafc90d45a35465e8 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/4829966e3d264b6dafc90d45a35465e8 2024-11-25T12:04:37,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/8a1d328198ff400fa884667f0cd4917f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/8a1d328198ff400fa884667f0cd4917f 2024-11-25T12:04:37,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/4d92d500a2d04c7faef954fe16e7f63e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/4d92d500a2d04c7faef954fe16e7f63e 2024-11-25T12:04:37,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/1cfacb0707664d99b07b913ec3350b86 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/1cfacb0707664d99b07b913ec3350b86 2024-11-25T12:04:37,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/bab8f05ab95d46a89d4ef0ea9bdaf140 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/bab8f05ab95d46a89d4ef0ea9bdaf140 2024-11-25T12:04:37,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/f1e3cf7d7b694e32a462cb441a82fcb4 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/f1e3cf7d7b694e32a462cb441a82fcb4 2024-11-25T12:04:37,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/47b9615aa4634ce8ab178e6b50992dc7 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/47b9615aa4634ce8ab178e6b50992dc7 2024-11-25T12:04:37,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/dc86c4a9b0824213a7c5e61672bef541 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/dc86c4a9b0824213a7c5e61672bef541 2024-11-25T12:04:37,144 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/ce30aa8125004788b348465b47e3f4b3 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/ce30aa8125004788b348465b47e3f4b3 2024-11-25T12:04:37,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/87a4228dade2496e84d3d244279140e4 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/87a4228dade2496e84d3d244279140e4 2024-11-25T12:04:37,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/63175b59da7b451fb6098f1fd4e70ea8 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/63175b59da7b451fb6098f1fd4e70ea8 2024-11-25T12:04:37,162 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/3a245cc4528b40aca2d2625b46acd7a9 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/3a245cc4528b40aca2d2625b46acd7a9 2024-11-25T12:04:37,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/afbe494045d84de0873f7fb110e05b36 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/afbe494045d84de0873f7fb110e05b36 2024-11-25T12:04:37,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/967ddf509eb945fc96086db61a51f291 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/967ddf509eb945fc96086db61a51f291 2024-11-25T12:04:37,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/2a4febebfd0e481899818be2fab9882f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/2a4febebfd0e481899818be2fab9882f 2024-11-25T12:04:37,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/7fdfdf72822a49679ad9342531e49ff2 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/7fdfdf72822a49679ad9342531e49ff2 2024-11-25T12:04:37,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/75ca074756aa40608b601b4451985af4 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/75ca074756aa40608b601b4451985af4 2024-11-25T12:04:37,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/9f8adedeb4734b6da40ad6ab33e9654e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/9f8adedeb4734b6da40ad6ab33e9654e 2024-11-25T12:04:37,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/241e4aab1b0144628d21dc1d09f29d52 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/241e4aab1b0144628d21dc1d09f29d52 2024-11-25T12:04:37,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/db86b847907946f09390c6daf6f99214 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/db86b847907946f09390c6daf6f99214 2024-11-25T12:04:37,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/3ea9ef9e080e41298d7fdba7f85bc6be to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/3ea9ef9e080e41298d7fdba7f85bc6be 2024-11-25T12:04:37,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/13dee5bb442d487aa0b84ae4d0b66e42 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/13dee5bb442d487aa0b84ae4d0b66e42 2024-11-25T12:04:37,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/1eec1b1fb1d049b4a1a7521ce9e90646 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/1eec1b1fb1d049b4a1a7521ce9e90646 2024-11-25T12:04:37,187 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/d2a6b7fe6de444f1b478805e1e6b4812 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/d2a6b7fe6de444f1b478805e1e6b4812 2024-11-25T12:04:37,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/58268b9073704bb69ffa4c9213fac5e3 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/58268b9073704bb69ffa4c9213fac5e3 2024-11-25T12:04:37,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/845707e599794c2397d70a77f7638c34, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/b4e206b0605e4e42ba47213400d77cb5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/357debbece1c46ceaeb5a2182c0ab622, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/374c692220c34235aa57fffccccd345a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/f88c1e824b3a434e86a8c8e971ae1403, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/002a10d0e2d9482299ecca0b37ff7fbe, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/09ea3eb0705e4ecf82d0592d0f737d88, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/bc97985f7f9b411ab28f1c3be1eb8368, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/f7e1ca5596104e95ba4629c28370a9d4, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/571e75caa7d9499e89d52dc53f391fe8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/b5e963689e364122b484f11eabe80f21, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/7f86f6486147481dac49702f3dc11c03, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/72608898eb2b4db4981fa0ef4fc6f01b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/7c8e3e7e730c4ef383e371c0333abaad, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/ba7d36e6d48149da87aeee1c70c9f6a8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/c691d6ebc56644229a39f4f56e2aac1d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/d4bcff66cb064223bbefa5054a58b14b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/758a9e2d6d764c72a836f7cc923e593e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/67528af7244742869587d2651f76475c, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/31cc30d02c8a4d4894326c7ed4e3ba2a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/6e9ddb810d53410a817cb843d8b09905, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/b5b02e2bbc6f4d48941c67295fc8ed44, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/18ff5030548546849c24b5b502139c82, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/cc2411fee4154d5d8f27fc1150bc56a0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/fc679f1e713e46e99b0c6637ef40ca81, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/ffb00da240074c4e91be5c795ba62d40] to archive 2024-11-25T12:04:37,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:04:37,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/845707e599794c2397d70a77f7638c34 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/845707e599794c2397d70a77f7638c34 2024-11-25T12:04:37,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/b4e206b0605e4e42ba47213400d77cb5 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/b4e206b0605e4e42ba47213400d77cb5 2024-11-25T12:04:37,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/357debbece1c46ceaeb5a2182c0ab622 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/357debbece1c46ceaeb5a2182c0ab622 2024-11-25T12:04:37,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/374c692220c34235aa57fffccccd345a to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/374c692220c34235aa57fffccccd345a 2024-11-25T12:04:37,215 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/f88c1e824b3a434e86a8c8e971ae1403 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/f88c1e824b3a434e86a8c8e971ae1403 2024-11-25T12:04:37,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/002a10d0e2d9482299ecca0b37ff7fbe to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/002a10d0e2d9482299ecca0b37ff7fbe 2024-11-25T12:04:37,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/09ea3eb0705e4ecf82d0592d0f737d88 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/09ea3eb0705e4ecf82d0592d0f737d88 2024-11-25T12:04:37,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/bc97985f7f9b411ab28f1c3be1eb8368 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/bc97985f7f9b411ab28f1c3be1eb8368 2024-11-25T12:04:37,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/f7e1ca5596104e95ba4629c28370a9d4 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/f7e1ca5596104e95ba4629c28370a9d4 2024-11-25T12:04:37,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/571e75caa7d9499e89d52dc53f391fe8 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/571e75caa7d9499e89d52dc53f391fe8 2024-11-25T12:04:37,230 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/b5e963689e364122b484f11eabe80f21 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/b5e963689e364122b484f11eabe80f21 2024-11-25T12:04:37,232 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/7f86f6486147481dac49702f3dc11c03 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/7f86f6486147481dac49702f3dc11c03 2024-11-25T12:04:37,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/72608898eb2b4db4981fa0ef4fc6f01b to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/72608898eb2b4db4981fa0ef4fc6f01b 2024-11-25T12:04:37,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/7c8e3e7e730c4ef383e371c0333abaad to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/7c8e3e7e730c4ef383e371c0333abaad 2024-11-25T12:04:37,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/ba7d36e6d48149da87aeee1c70c9f6a8 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/ba7d36e6d48149da87aeee1c70c9f6a8 2024-11-25T12:04:37,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/c691d6ebc56644229a39f4f56e2aac1d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/c691d6ebc56644229a39f4f56e2aac1d 2024-11-25T12:04:37,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/d4bcff66cb064223bbefa5054a58b14b to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/d4bcff66cb064223bbefa5054a58b14b 2024-11-25T12:04:37,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/758a9e2d6d764c72a836f7cc923e593e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/758a9e2d6d764c72a836f7cc923e593e 2024-11-25T12:04:37,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/67528af7244742869587d2651f76475c to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/67528af7244742869587d2651f76475c 2024-11-25T12:04:37,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/31cc30d02c8a4d4894326c7ed4e3ba2a to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/31cc30d02c8a4d4894326c7ed4e3ba2a 2024-11-25T12:04:37,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/6e9ddb810d53410a817cb843d8b09905 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/6e9ddb810d53410a817cb843d8b09905 2024-11-25T12:04:37,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/b5b02e2bbc6f4d48941c67295fc8ed44 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/b5b02e2bbc6f4d48941c67295fc8ed44 2024-11-25T12:04:37,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/18ff5030548546849c24b5b502139c82 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/18ff5030548546849c24b5b502139c82 2024-11-25T12:04:37,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/cc2411fee4154d5d8f27fc1150bc56a0 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/cc2411fee4154d5d8f27fc1150bc56a0 2024-11-25T12:04:37,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/fc679f1e713e46e99b0c6637ef40ca81 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/fc679f1e713e46e99b0c6637ef40ca81 2024-11-25T12:04:37,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/ffb00da240074c4e91be5c795ba62d40 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/ffb00da240074c4e91be5c795ba62d40 2024-11-25T12:04:37,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/45dbd651deed41e09bfd1fc55d827ac2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/feb2ac6c5a3e48e2bd36ce6174ea2a6d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/347cd8818dec4379a2f0bbc0b27f94f8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/426966af2a9540b18d3fbb8bcaf076cd, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/61f2e05c6d75485eb1a4e094573a2180, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/b6ce9d610a5841c18b2f6f4509f03a5d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/3bb02f08f06d45ed9777f7aa4f732fa8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/a5ea9abe76034f7c81c2b1494acaf4ef, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/32bd7d06a66e413297100e1b7e7d1512, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/2564ae70ce394e86a59dec8616bf63b5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/fd84a9ce512440bc968b729430d33754, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/6384c21f767446a8a5f3e5e872bee1c5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/b21bfbedd2b44ebc930fb75140c297e8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/03c6d5d8b31942f98e98c07e23373f0a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/8ffc50fc96fc43ccbea1411ef3587d24, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/bfa23d210bb44d1cbe25ae537a8e7753, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/613e60bae36743918e5d6d162efd1f2b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/786919e3925f4796a7324305ba642c49, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/8324ca97fefc42eb897aafba88cc6c50, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/1e7928509a704d45bb92001cd46e4d6f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/aa58d95741704e2cbebc7ead7b9e7f24, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/0a56160977da4b768a198a34a2cc0a48, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/a603ee27a00e4de9b149e027c0caa492, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/5228c4c61e40474192d131d45221e0b0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/2709da7fe27943a68621e8c5d32108f6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/1cc6febb1b934f639249454477731d5b] to archive 2024-11-25T12:04:37,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:04:37,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/45dbd651deed41e09bfd1fc55d827ac2 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/45dbd651deed41e09bfd1fc55d827ac2 2024-11-25T12:04:37,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/feb2ac6c5a3e48e2bd36ce6174ea2a6d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/feb2ac6c5a3e48e2bd36ce6174ea2a6d 2024-11-25T12:04:37,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/347cd8818dec4379a2f0bbc0b27f94f8 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/347cd8818dec4379a2f0bbc0b27f94f8 2024-11-25T12:04:37,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/426966af2a9540b18d3fbb8bcaf076cd to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/426966af2a9540b18d3fbb8bcaf076cd 2024-11-25T12:04:37,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/61f2e05c6d75485eb1a4e094573a2180 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/61f2e05c6d75485eb1a4e094573a2180 2024-11-25T12:04:37,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/b6ce9d610a5841c18b2f6f4509f03a5d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/b6ce9d610a5841c18b2f6f4509f03a5d 2024-11-25T12:04:37,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/3bb02f08f06d45ed9777f7aa4f732fa8 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/3bb02f08f06d45ed9777f7aa4f732fa8 2024-11-25T12:04:37,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/a5ea9abe76034f7c81c2b1494acaf4ef to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/a5ea9abe76034f7c81c2b1494acaf4ef 2024-11-25T12:04:37,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/32bd7d06a66e413297100e1b7e7d1512 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/32bd7d06a66e413297100e1b7e7d1512 2024-11-25T12:04:37,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/2564ae70ce394e86a59dec8616bf63b5 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/2564ae70ce394e86a59dec8616bf63b5 2024-11-25T12:04:37,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/fd84a9ce512440bc968b729430d33754 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/fd84a9ce512440bc968b729430d33754 2024-11-25T12:04:37,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/6384c21f767446a8a5f3e5e872bee1c5 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/6384c21f767446a8a5f3e5e872bee1c5 2024-11-25T12:04:37,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/b21bfbedd2b44ebc930fb75140c297e8 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/b21bfbedd2b44ebc930fb75140c297e8 2024-11-25T12:04:37,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/03c6d5d8b31942f98e98c07e23373f0a to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/03c6d5d8b31942f98e98c07e23373f0a 2024-11-25T12:04:37,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/8ffc50fc96fc43ccbea1411ef3587d24 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/8ffc50fc96fc43ccbea1411ef3587d24 2024-11-25T12:04:37,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/bfa23d210bb44d1cbe25ae537a8e7753 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/bfa23d210bb44d1cbe25ae537a8e7753 2024-11-25T12:04:37,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/613e60bae36743918e5d6d162efd1f2b to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/613e60bae36743918e5d6d162efd1f2b 2024-11-25T12:04:37,376 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/786919e3925f4796a7324305ba642c49 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/786919e3925f4796a7324305ba642c49 2024-11-25T12:04:37,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/8324ca97fefc42eb897aafba88cc6c50 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/8324ca97fefc42eb897aafba88cc6c50 2024-11-25T12:04:37,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/1e7928509a704d45bb92001cd46e4d6f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/1e7928509a704d45bb92001cd46e4d6f 2024-11-25T12:04:37,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/aa58d95741704e2cbebc7ead7b9e7f24 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/aa58d95741704e2cbebc7ead7b9e7f24 2024-11-25T12:04:37,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/0a56160977da4b768a198a34a2cc0a48 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/0a56160977da4b768a198a34a2cc0a48 2024-11-25T12:04:37,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/a603ee27a00e4de9b149e027c0caa492 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/a603ee27a00e4de9b149e027c0caa492 2024-11-25T12:04:37,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/5228c4c61e40474192d131d45221e0b0 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/5228c4c61e40474192d131d45221e0b0 2024-11-25T12:04:37,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/2709da7fe27943a68621e8c5d32108f6 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/2709da7fe27943a68621e8c5d32108f6 2024-11-25T12:04:37,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/1cc6febb1b934f639249454477731d5b to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/1cc6febb1b934f639249454477731d5b 2024-11-25T12:04:37,448 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/recovered.edits/387.seqid, newMaxSeqId=387, maxSeqId=4 2024-11-25T12:04:37,453 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. 2024-11-25T12:04:37,453 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for 14ed09f1e402e2ed659ccb58744c29bb: Waiting for close lock at 1732536276125Running coprocessor pre-close hooks at 1732536276125Disabling compacts and flushes for region at 1732536276125Disabling writes for close at 1732536276125Obtaining lock to block concurrent updates at 1732536276125Preparing flush snapshotting stores in 14ed09f1e402e2ed659ccb58744c29bb at 1732536276125Finished memstore snapshotting TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732536276125Flushing stores of TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb. at 1732536276126 (+1 ms)Flushing 14ed09f1e402e2ed659ccb58744c29bb/A: creating writer at 1732536276126Flushing mob file 14ed09f1e402e2ed659ccb58744c29bb/A: appending metadata at 1732536276148 (+22 ms)Flushing mob file 14ed09f1e402e2ed659ccb58744c29bb/A: closing flushed file at 1732536276148Flushing 14ed09f1e402e2ed659ccb58744c29bb/A: appending metadata at 1732536276574 (+426 ms)Flushing 14ed09f1e402e2ed659ccb58744c29bb/A: closing flushed file at 1732536276574Flushing 14ed09f1e402e2ed659ccb58744c29bb/B: creating writer at 1732536276590 (+16 ms)Flushing 14ed09f1e402e2ed659ccb58744c29bb/B: appending metadata at 1732536276593 (+3 ms)Flushing 14ed09f1e402e2ed659ccb58744c29bb/B: closing flushed file at 1732536276593Flushing 14ed09f1e402e2ed659ccb58744c29bb/C: creating writer at 1732536276612 (+19 ms)Flushing 14ed09f1e402e2ed659ccb58744c29bb/C: appending metadata at 1732536276620 (+8 ms)Flushing 14ed09f1e402e2ed659ccb58744c29bb/C: closing flushed file at 1732536276620Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6752e888: reopening flushed file at 1732536277058 (+438 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@724bc255: reopening flushed file at 1732536277068 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31e819be: reopening flushed file at 1732536277074 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 14ed09f1e402e2ed659ccb58744c29bb in 971ms, sequenceid=384, compaction requested=false at 1732536277097 (+23 ms)Writing region close event to WAL at 1732536277426 (+329 ms)Running coprocessor post-close hooks at 1732536277453 (+27 ms)Closed at 1732536277453 2024-11-25T12:04:37,458 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed 14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:37,463 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=14ed09f1e402e2ed659ccb58744c29bb, regionState=CLOSED 2024-11-25T12:04:37,465 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 because future has completed 2024-11-25T12:04:37,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-25T12:04:37,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure 14ed09f1e402e2ed659ccb58744c29bb, server=2bf237fd86ca,37643,1732536214674 in 1.4990 sec 2024-11-25T12:04:37,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-25T12:04:37,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ed09f1e402e2ed659ccb58744c29bb, UNASSIGN in 1.5060 sec 2024-11-25T12:04:37,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-25T12:04:37,477 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5110 sec 2024-11-25T12:04:37,479 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536277478"}]},"ts":"1732536277478"} 2024-11-25T12:04:37,481 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T12:04:37,481 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T12:04:37,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5290 sec 2024-11-25T12:04:38,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-25T12:04:38,107 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:38,107 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-25T12:04:38,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:04:38,110 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:04:38,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T12:04:38,111 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:04:38,113 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,117 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/recovered.edits] 2024-11-25T12:04:38,122 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/1edb00abbd6c419c92b0afcf07054578 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/1edb00abbd6c419c92b0afcf07054578 2024-11-25T12:04:38,126 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/2123afef12874ffab8fb7cc5ab87f1fd to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/A/2123afef12874ffab8fb7cc5ab87f1fd 2024-11-25T12:04:38,129 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/183bc7b831a94524acf349dfc169c6df to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/183bc7b831a94524acf349dfc169c6df 2024-11-25T12:04:38,131 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/779c894874b34dc184729c496eec3c88 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/B/779c894874b34dc184729c496eec3c88 2024-11-25T12:04:38,143 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/ac5c52ffc2634f59b5925e010cd9138f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/ac5c52ffc2634f59b5925e010cd9138f 2024-11-25T12:04:38,145 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/c20b6259837647f6b79e61a146922468 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/C/c20b6259837647f6b79e61a146922468 2024-11-25T12:04:38,163 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/recovered.edits/387.seqid to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb/recovered.edits/387.seqid 2024-11-25T12:04:38,166 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,166 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T12:04:38,166 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T12:04:38,168 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-25T12:04:38,176 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250149b3139029462fa5a6b41169f432da_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250149b3139029462fa5a6b41169f432da_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,180 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253d1da01a3d694fee81a7d4022063c509_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253d1da01a3d694fee81a7d4022063c509_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,189 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254304aa3abd914825aa0e3d1d8de423a1_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254304aa3abd914825aa0e3d1d8de423a1_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,198 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112554e363e2e4a3480b9ec4ca92d29b092b_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112554e363e2e4a3480b9ec4ca92d29b092b_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,206 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256651dbfb9d9842b09c4d585a020b56a7_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256651dbfb9d9842b09c4d585a020b56a7_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,210 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125734aa8d7de1b479bb8d02b9734f78c99_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125734aa8d7de1b479bb8d02b9734f78c99_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,211 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112582bae34884f34a90b5c682d0d2616b16_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112582bae34884f34a90b5c682d0d2616b16_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,212 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125844d1cf3d4914ba18b1c012a56df0496_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125844d1cf3d4914ba18b1c012a56df0496_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,214 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112584c97371f8c342d682627a8a995d9b44_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112584c97371f8c342d682627a8a995d9b44_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T12:04:38,217 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112585f9c85e8c704299a3d6624a37935153_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112585f9c85e8c704299a3d6624a37935153_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,218 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112594b8c020074f435192869b6b58f4f4f0_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112594b8c020074f435192869b6b58f4f4f0_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,220 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259e2770da8e2a47d9827a033824235965_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259e2770da8e2a47d9827a033824235965_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,222 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ae9661048b3b41fcbc5dc07337bbfd96_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ae9661048b3b41fcbc5dc07337bbfd96_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,223 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c6681523879f40eab16ca6af35416a75_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c6681523879f40eab16ca6af35416a75_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,224 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c7a06c9b1abb4126bb52048be14658ad_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c7a06c9b1abb4126bb52048be14658ad_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,226 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d453bb53f02d46e9b9207182132626b3_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d453bb53f02d46e9b9207182132626b3_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,227 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d4fe02c36df34b8696d9a97c5bc2c6de_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d4fe02c36df34b8696d9a97c5bc2c6de_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,229 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e1c9152269a44b558f1cb1e3558a0bd7_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e1c9152269a44b558f1cb1e3558a0bd7_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,230 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e22c6fb6523048e8b6d460523d81022e_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125e22c6fb6523048e8b6d460523d81022e_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,232 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ec1d8eff9bb64faf8af1ea620ef37ccf_14ed09f1e402e2ed659ccb58744c29bb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ec1d8eff9bb64faf8af1ea620ef37ccf_14ed09f1e402e2ed659ccb58744c29bb 2024-11-25T12:04:38,233 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T12:04:38,241 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:04:38,249 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T12:04:38,251 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T12:04:38,253 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:04:38,253 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T12:04:38,253 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732536278253"}]},"ts":"9223372036854775807"} 2024-11-25T12:04:38,256 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T12:04:38,256 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 14ed09f1e402e2ed659ccb58744c29bb, NAME => 'TestAcidGuarantees,,1732536245432.14ed09f1e402e2ed659ccb58744c29bb.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T12:04:38,256 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T12:04:38,256 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732536278256"}]},"ts":"9223372036854775807"} 2024-11-25T12:04:38,258 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T12:04:38,259 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:04:38,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 151 msec 2024-11-25T12:04:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-25T12:04:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T12:04:38,429 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:38,442 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 238), OpenFileDescriptor=453 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1338 (was 1076) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4657 (was 4791) 2024-11-25T12:04:38,452 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=1338, ProcessCount=11, AvailableMemoryMB=4657 2024-11-25T12:04:38,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T12:04:38,454 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:04:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T12:04:38,456 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T12:04:38,456 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:38,456 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-11-25T12:04:38,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-25T12:04:38,457 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T12:04:38,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742048_1224 (size=959) 2024-11-25T12:04:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-25T12:04:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-25T12:04:38,867 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d 2024-11-25T12:04:38,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742049_1225 (size=53) 2024-11-25T12:04:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-25T12:04:39,283 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:04:39,283 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5f0aa52f88963b28a40fb54be46c7463, disabling compactions & flushes 2024-11-25T12:04:39,283 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:39,283 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:39,283 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. after waiting 0 ms 2024-11-25T12:04:39,283 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:39,283 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:39,283 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5f0aa52f88963b28a40fb54be46c7463: Waiting for close lock at 1732536279283Disabling compacts and flushes for region at 1732536279283Disabling writes for close at 1732536279283Writing region close event to WAL at 1732536279283Closed at 1732536279283 2024-11-25T12:04:39,286 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T12:04:39,286 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732536279286"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732536279286"}]},"ts":"1732536279286"} 2024-11-25T12:04:39,288 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T12:04:39,289 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T12:04:39,289 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536279289"}]},"ts":"1732536279289"} 2024-11-25T12:04:39,290 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T12:04:39,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f0aa52f88963b28a40fb54be46c7463, ASSIGN}] 2024-11-25T12:04:39,292 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f0aa52f88963b28a40fb54be46c7463, ASSIGN 2024-11-25T12:04:39,293 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f0aa52f88963b28a40fb54be46c7463, ASSIGN; state=OFFLINE, location=2bf237fd86ca,37643,1732536214674; forceNewPlan=false, retain=false 2024-11-25T12:04:39,444 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=5f0aa52f88963b28a40fb54be46c7463, regionState=OPENING, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:39,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f0aa52f88963b28a40fb54be46c7463, ASSIGN because future has completed 2024-11-25T12:04:39,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674}] 2024-11-25T12:04:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-25T12:04:39,610 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:39,610 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:04:39,611 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:39,611 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:04:39,611 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:39,611 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:39,613 INFO [StoreOpener-5f0aa52f88963b28a40fb54be46c7463-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:39,625 INFO [StoreOpener-5f0aa52f88963b28a40fb54be46c7463-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f0aa52f88963b28a40fb54be46c7463 columnFamilyName A 2024-11-25T12:04:39,625 DEBUG [StoreOpener-5f0aa52f88963b28a40fb54be46c7463-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:39,627 INFO [StoreOpener-5f0aa52f88963b28a40fb54be46c7463-1 {}] regionserver.HStore(327): Store=5f0aa52f88963b28a40fb54be46c7463/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:04:39,627 INFO [StoreOpener-5f0aa52f88963b28a40fb54be46c7463-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:39,633 INFO [StoreOpener-5f0aa52f88963b28a40fb54be46c7463-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f0aa52f88963b28a40fb54be46c7463 columnFamilyName B 2024-11-25T12:04:39,633 DEBUG [StoreOpener-5f0aa52f88963b28a40fb54be46c7463-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:39,636 INFO [StoreOpener-5f0aa52f88963b28a40fb54be46c7463-1 {}] regionserver.HStore(327): Store=5f0aa52f88963b28a40fb54be46c7463/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:04:39,637 INFO [StoreOpener-5f0aa52f88963b28a40fb54be46c7463-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:39,644 INFO [StoreOpener-5f0aa52f88963b28a40fb54be46c7463-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f0aa52f88963b28a40fb54be46c7463 columnFamilyName C 2024-11-25T12:04:39,644 DEBUG [StoreOpener-5f0aa52f88963b28a40fb54be46c7463-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:04:39,645 INFO [StoreOpener-5f0aa52f88963b28a40fb54be46c7463-1 {}] regionserver.HStore(327): Store=5f0aa52f88963b28a40fb54be46c7463/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:04:39,646 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:39,646 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:39,648 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:39,650 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:39,651 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:39,651 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:04:39,660 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:39,667 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:04:39,669 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 5f0aa52f88963b28a40fb54be46c7463; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75128016, jitterRate=0.11949467658996582}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:04:39,669 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:39,671 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 5f0aa52f88963b28a40fb54be46c7463: Running coprocessor pre-open hook at 1732536279611Writing region info on filesystem at 1732536279611Initializing all the Stores at 1732536279612 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536279612Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536279613 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536279613Cleaning up temporary data from old regions at 1732536279651 (+38 ms)Running coprocessor post-open hooks at 1732536279669 (+18 ms)Region opened successfully at 1732536279671 (+2 ms) 2024-11-25T12:04:39,672 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., pid=59, masterSystemTime=1732536279606 2024-11-25T12:04:39,678 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:39,678 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:39,680 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=5f0aa52f88963b28a40fb54be46c7463, regionState=OPEN, openSeqNum=2, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:39,682 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 because future has completed 2024-11-25T12:04:39,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-25T12:04:39,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure 5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 in 231 msec 2024-11-25T12:04:39,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-25T12:04:39,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f0aa52f88963b28a40fb54be46c7463, ASSIGN in 402 msec 2024-11-25T12:04:39,697 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T12:04:39,698 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536279697"}]},"ts":"1732536279697"} 2024-11-25T12:04:39,699 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T12:04:39,701 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T12:04:39,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2470 sec 2024-11-25T12:04:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-25T12:04:40,606 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:40,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39edb4ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:40,608 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:40,609 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56535f13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:40,611 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fb3392, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,612 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:40,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:40,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:40,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:40,616 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T12:04:40,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 2bf237fd86ca,40019,1732536213609 2024-11-25T12:04:40,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@44fb4f9f 2024-11-25T12:04:40,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T12:04:40,617 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T12:04:40,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253bc1fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,619 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:40,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:40,620 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:40,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:40,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:40,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45ffff46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:40,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:40,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:40,621 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37878, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:40,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eee6f5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16ea2060, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,623 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:40,623 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:40,623 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:40,624 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:40,624 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:40,624 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@933a0a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,624 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:40,624 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:40,624 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:40,625 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:40,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cfe7e51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6107429, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,626 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:40,627 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:40,627 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54cabf84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:40,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:40,629 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:40,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@488534d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29ad0166, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:40,633 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:40,633 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57337ef2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:40,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:40,636 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:40,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@617df823, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a4178ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:40,648 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:40,650 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:40,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:40,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:40,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e2da0eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:40,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:40,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:40,651 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:40,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4708d529, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad08ddd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,653 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:40,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:40,654 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a1767aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:40,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:40,656 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:40,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8794a9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@542fb97c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,661 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:40,661 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:40,662 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33aae1d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:40,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:40,663 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:40,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1926bc7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c46d298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:40,665 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:40,666 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:40,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@358beaa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:40,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:40,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:40,668 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:40,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa7e9ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f5167d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:40,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:40,671 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:40,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:40,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:40,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a55e063, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:40,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:40,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:40,672 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:40,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343e1a89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a80d28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:04:40,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:04:40,682 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:04:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:04:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:04:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e10012d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:04:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:04:40,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:04:40,683 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:04:40,684 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@112f32ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:04:40,688 DEBUG [Thread-1060 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:40,689 DEBUG [Thread-1062 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:40,690 DEBUG [Thread-1060 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:40,690 DEBUG [Thread-1060 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:40,694 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:40,695 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47558, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:40,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:04:40,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:04:40,703 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:04:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-25T12:04:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T12:04:40,705 DEBUG [Thread-1079 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:40,706 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:04:40,706 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:04:40,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:04:40,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:40,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:40,716 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:40,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:04:40,722 DEBUG [Thread-1068 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:40,723 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:40,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:04:40,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:40,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:40,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:40,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:04:40,734 DEBUG [Thread-1071 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:40,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:40,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:40,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:40,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:04:40,749 DEBUG [Thread-1073 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:40,753 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:40,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:04:40,760 DEBUG [Thread-1077 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:40,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:40,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:40,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:40,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:40,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536340762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:40,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:40,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:40,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:40,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:04:40,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:40,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536340764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:40,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:40,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:40,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536340767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:40,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:40,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:40,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:40,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/e7b6217967ef4cedb1edd0fd8c5ac47f is 50, key is test_row_0/A:col10/1732536280720/Put/seqid=0 2024-11-25T12:04:40,772 DEBUG [Thread-1064 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:40,773 DEBUG [Thread-1066 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:40,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:40,774 DEBUG [Thread-1075 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:04:40,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:40,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:40,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:40,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:40,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:04:40,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:04:40,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:04:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:04:40,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:04:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:04:40,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:40,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536340779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:40,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:40,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:40,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:40,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:40,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47622 deadline: 1732536340781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:40,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742050_1226 (size=9657) 2024-11-25T12:04:40,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/e7b6217967ef4cedb1edd0fd8c5ac47f 2024-11-25T12:04:40,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T12:04:40,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/b52b872dd24e4e54a8703af519aba38e is 50, key is test_row_0/B:col10/1732536280720/Put/seqid=0 2024-11-25T12:04:40,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T12:04:40,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:40,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:40,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:40,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:40,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:40,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:40,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536340866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:40,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742051_1227 (size=9657) 2024-11-25T12:04:40,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:40,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536340878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:40,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:40,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:40,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:40,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536340884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:40,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47622 deadline: 1732536340886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:40,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:40,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536340888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:40,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:40,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:40,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:40,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:41,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T12:04:41,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T12:04:41,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:41,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:41,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:41,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:41,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:41,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:41,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536341087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:41,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:41,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536341097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:41,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:41,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536341106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:41,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:41,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536341106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:41,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:41,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:41,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47622 deadline: 1732536341118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:41,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:41,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T12:04:41,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:41,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:41,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:41,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:41,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:41,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:41,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/b52b872dd24e4e54a8703af519aba38e 2024-11-25T12:04:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T12:04:41,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T12:04:41,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:41,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:41,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:41,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:41,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:41,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/57d1bedabf8642ac982c6ac441a5d1a8 is 50, key is test_row_0/C:col10/1732536280720/Put/seqid=0 2024-11-25T12:04:41,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742052_1228 (size=9657) 2024-11-25T12:04:41,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/57d1bedabf8642ac982c6ac441a5d1a8 2024-11-25T12:04:41,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536341406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:41,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:41,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536341407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:41,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/e7b6217967ef4cedb1edd0fd8c5ac47f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/e7b6217967ef4cedb1edd0fd8c5ac47f 2024-11-25T12:04:41,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:41,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536341416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:41,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:41,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536341417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:41,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:41,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/e7b6217967ef4cedb1edd0fd8c5ac47f, entries=100, sequenceid=12, filesize=9.4 K 2024-11-25T12:04:41,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/b52b872dd24e4e54a8703af519aba38e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/b52b872dd24e4e54a8703af519aba38e 2024-11-25T12:04:41,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47622 deadline: 1732536341436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:41,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/b52b872dd24e4e54a8703af519aba38e, entries=100, sequenceid=12, filesize=9.4 K 2024-11-25T12:04:41,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/57d1bedabf8642ac982c6ac441a5d1a8 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/57d1bedabf8642ac982c6ac441a5d1a8 2024-11-25T12:04:41,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/57d1bedabf8642ac982c6ac441a5d1a8, entries=100, sequenceid=12, filesize=9.4 K 2024-11-25T12:04:41,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 5f0aa52f88963b28a40fb54be46c7463 in 780ms, sequenceid=12, compaction requested=false 2024-11-25T12:04:41,503 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-25T12:04:41,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:41,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-25T12:04:41,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:41,519 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-25T12:04:41,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/65e44d459dae4adbae5d980c7b44ca81 is 50, key is test_row_0/A:col10/1732536280757/Put/seqid=0 2024-11-25T12:04:41,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742053_1229 (size=12001) 2024-11-25T12:04:41,590 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/65e44d459dae4adbae5d980c7b44ca81 2024-11-25T12:04:41,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/edf0f3640c3f49c099949c7860169326 is 50, key is test_row_0/B:col10/1732536280757/Put/seqid=0 2024-11-25T12:04:41,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742054_1230 (size=12001) 2024-11-25T12:04:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T12:04:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:41,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:41,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:41,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47622 deadline: 1732536341948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:41,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536341950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:41,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536341950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:41,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:41,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536341953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:41,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536341955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:41,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:41,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:42,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536342066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:42,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:42,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536342066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:42,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:42,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536342067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:42,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,071 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/edf0f3640c3f49c099949c7860169326 2024-11-25T12:04:42,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:42,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:42,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536342069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:42,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/3f479537cfb0438a845eba912f0ab372 is 50, key is test_row_0/C:col10/1732536280757/Put/seqid=0 2024-11-25T12:04:42,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742055_1231 (size=12001) 2024-11-25T12:04:42,130 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/3f479537cfb0438a845eba912f0ab372 2024-11-25T12:04:42,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/65e44d459dae4adbae5d980c7b44ca81 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/65e44d459dae4adbae5d980c7b44ca81 2024-11-25T12:04:42,156 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/65e44d459dae4adbae5d980c7b44ca81, entries=150, sequenceid=38, filesize=11.7 K 2024-11-25T12:04:42,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/edf0f3640c3f49c099949c7860169326 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/edf0f3640c3f49c099949c7860169326 2024-11-25T12:04:42,168 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/edf0f3640c3f49c099949c7860169326, entries=150, sequenceid=38, filesize=11.7 K 2024-11-25T12:04:42,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/3f479537cfb0438a845eba912f0ab372 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/3f479537cfb0438a845eba912f0ab372 2024-11-25T12:04:42,186 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/3f479537cfb0438a845eba912f0ab372, entries=150, sequenceid=38, filesize=11.7 K 2024-11-25T12:04:42,199 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 5f0aa52f88963b28a40fb54be46c7463 in 680ms, sequenceid=38, compaction requested=false 2024-11-25T12:04:42,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:42,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:42,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-25T12:04:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-25T12:04:42,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-25T12:04:42,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4950 sec 2024-11-25T12:04:42,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.5030 sec 2024-11-25T12:04:42,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:42,278 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:42,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:04:42,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/906c216daa284a959b4ae2f5b3eed5c0 is 50, key is test_row_0/A:col10/1732536282278/Put/seqid=0 2024-11-25T12:04:42,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:42,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536342337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:42,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536342338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:42,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:42,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536342340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:42,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536342342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742056_1232 (size=14341) 2024-11-25T12:04:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:42,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/906c216daa284a959b4ae2f5b3eed5c0 2024-11-25T12:04:42,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/bad01e1a938a4c91bf8bce03b8330e69 is 50, key is test_row_0/B:col10/1732536282278/Put/seqid=0 2024-11-25T12:04:42,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536342446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:42,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:42,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536342457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:42,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:42,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536342457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:42,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:42,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536342461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:42,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742057_1233 (size=12001) 2024-11-25T12:04:42,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/bad01e1a938a4c91bf8bce03b8330e69 2024-11-25T12:04:42,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/10ecff3a013f4497adc25668a2611adb is 50, key is test_row_0/C:col10/1732536282278/Put/seqid=0 2024-11-25T12:04:42,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742058_1234 (size=12001) 2024-11-25T12:04:42,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536342660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:42,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:42,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536342667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:42,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:42,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536342667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:42,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:42,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536342679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:42,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:42,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-25T12:04:42,932 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:42,934 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:04:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-25T12:04:42,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T12:04:42,945 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:04:42,947 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:04:42,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:04:42,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47622 deadline: 1732536342970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:42,975 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_1' in 5f0aa52f88963b28a40fb54be46c7463 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:04:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:42,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536342977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:42,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536342987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:42,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536342987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:42,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:43,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536342997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:43,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:43,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/10ecff3a013f4497adc25668a2611adb 2024-11-25T12:04:43,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/906c216daa284a959b4ae2f5b3eed5c0 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/906c216daa284a959b4ae2f5b3eed5c0 2024-11-25T12:04:43,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/906c216daa284a959b4ae2f5b3eed5c0, entries=200, sequenceid=50, filesize=14.0 K 2024-11-25T12:04:43,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/bad01e1a938a4c91bf8bce03b8330e69 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/bad01e1a938a4c91bf8bce03b8330e69 2024-11-25T12:04:43,034 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T12:04:43,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/bad01e1a938a4c91bf8bce03b8330e69, entries=150, sequenceid=50, filesize=11.7 K 2024-11-25T12:04:43,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/10ecff3a013f4497adc25668a2611adb as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/10ecff3a013f4497adc25668a2611adb 2024-11-25T12:04:43,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/10ecff3a013f4497adc25668a2611adb, entries=150, sequenceid=50, filesize=11.7 K 2024-11-25T12:04:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T12:04:43,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5f0aa52f88963b28a40fb54be46c7463 in 779ms, sequenceid=50, compaction requested=true 2024-11-25T12:04:43,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:43,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:04:43,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:43,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:04:43,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:04:43,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:04:43,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:04:43,057 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:43,058 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:43,059 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:43,059 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/C is initiating minor compaction (all files) 2024-11-25T12:04:43,059 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/C in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:43,060 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/57d1bedabf8642ac982c6ac441a5d1a8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/3f479537cfb0438a845eba912f0ab372, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/10ecff3a013f4497adc25668a2611adb] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=32.9 K 2024-11-25T12:04:43,060 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:43,060 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/A is initiating minor compaction (all files) 2024-11-25T12:04:43,060 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/A in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:43,060 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/e7b6217967ef4cedb1edd0fd8c5ac47f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/65e44d459dae4adbae5d980c7b44ca81, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/906c216daa284a959b4ae2f5b3eed5c0] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=35.2 K 2024-11-25T12:04:43,061 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7b6217967ef4cedb1edd0fd8c5ac47f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732536280713 2024-11-25T12:04:43,061 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 57d1bedabf8642ac982c6ac441a5d1a8, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732536280713 2024-11-25T12:04:43,061 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65e44d459dae4adbae5d980c7b44ca81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732536280757 2024-11-25T12:04:43,062 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f479537cfb0438a845eba912f0ab372, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732536280757 2024-11-25T12:04:43,062 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 906c216daa284a959b4ae2f5b3eed5c0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732536281941 2024-11-25T12:04:43,062 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 10ecff3a013f4497adc25668a2611adb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732536281941 2024-11-25T12:04:43,088 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#A#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:43,089 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/712e028a104a43a6ab5d04cd6f9a36eb is 50, key is test_row_0/A:col10/1732536282278/Put/seqid=0 2024-11-25T12:04:43,095 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#C#compaction#190 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:43,096 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/c2b7160087aa471296964e42aba06125 is 50, key is test_row_0/C:col10/1732536282278/Put/seqid=0 2024-11-25T12:04:43,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-25T12:04:43,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:43,109 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T12:04:43,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/5dc7e23630a3495b8466611bbe7d5152 is 50, key is test_row_0/A:col10/1732536282340/Put/seqid=0 2024-11-25T12:04:43,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742059_1235 (size=12104) 2024-11-25T12:04:43,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742060_1236 (size=12104) 2024-11-25T12:04:43,177 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/712e028a104a43a6ab5d04cd6f9a36eb as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/712e028a104a43a6ab5d04cd6f9a36eb 2024-11-25T12:04:43,191 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/A of 5f0aa52f88963b28a40fb54be46c7463 into 712e028a104a43a6ab5d04cd6f9a36eb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:43,191 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:43,191 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/A, priority=13, startTime=1732536283057; duration=0sec 2024-11-25T12:04:43,191 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:43,192 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:A 2024-11-25T12:04:43,192 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:43,194 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:43,194 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/B is initiating minor compaction (all files) 2024-11-25T12:04:43,194 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/B in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:43,195 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/b52b872dd24e4e54a8703af519aba38e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/edf0f3640c3f49c099949c7860169326, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/bad01e1a938a4c91bf8bce03b8330e69] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=32.9 K 2024-11-25T12:04:43,195 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting b52b872dd24e4e54a8703af519aba38e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732536280713 2024-11-25T12:04:43,196 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting edf0f3640c3f49c099949c7860169326, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732536280757 2024-11-25T12:04:43,196 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting bad01e1a938a4c91bf8bce03b8330e69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732536281941 2024-11-25T12:04:43,219 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#B#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:43,220 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/6885fae639854f0190dfee976e8ec6a8 is 50, key is test_row_0/B:col10/1732536282278/Put/seqid=0 2024-11-25T12:04:43,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742061_1237 (size=12001) 2024-11-25T12:04:43,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742062_1238 (size=12104) 2024-11-25T12:04:43,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T12:04:43,277 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/6885fae639854f0190dfee976e8ec6a8 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/6885fae639854f0190dfee976e8ec6a8 2024-11-25T12:04:43,291 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/B of 5f0aa52f88963b28a40fb54be46c7463 into 6885fae639854f0190dfee976e8ec6a8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:43,291 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:43,291 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/B, priority=13, startTime=1732536283057; duration=0sec 2024-11-25T12:04:43,291 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:43,291 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:B 2024-11-25T12:04:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:43,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:43,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:43,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536343524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:43,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:43,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536343527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:43,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536343531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:43,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:43,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536343533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:43,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:43,586 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/c2b7160087aa471296964e42aba06125 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/c2b7160087aa471296964e42aba06125 2024-11-25T12:04:43,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T12:04:43,595 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/C of 5f0aa52f88963b28a40fb54be46c7463 into c2b7160087aa471296964e42aba06125(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:43,595 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:43,596 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/C, priority=13, startTime=1732536283057; duration=0sec 2024-11-25T12:04:43,596 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:43,596 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:C 2024-11-25T12:04:43,627 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/5dc7e23630a3495b8466611bbe7d5152 2024-11-25T12:04:43,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536343647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:43,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536343647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:43,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536343647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:43,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:43,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536343647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:43,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:43,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/13d45ce6665d4cda93781c97052b6027 is 50, key is test_row_0/B:col10/1732536282340/Put/seqid=0 2024-11-25T12:04:43,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:43,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:43,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742063_1239 (size=12001) 2024-11-25T12:04:43,701 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/13d45ce6665d4cda93781c97052b6027 2024-11-25T12:04:43,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/263db9617fe240ea98175161e49ad131 is 50, key is test_row_0/C:col10/1732536282340/Put/seqid=0 2024-11-25T12:04:43,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742064_1240 (size=12001) 2024-11-25T12:04:43,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536343869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:43,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536343869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:43,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:43,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:43,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536343879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:43,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:43,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536343879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:43,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:43,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T12:04:44,159 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/263db9617fe240ea98175161e49ad131 2024-11-25T12:04:44,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536344187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:44,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536344186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:44,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/5dc7e23630a3495b8466611bbe7d5152 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/5dc7e23630a3495b8466611bbe7d5152 2024-11-25T12:04:44,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:44,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536344197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536344198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,207 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:44,215 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/5dc7e23630a3495b8466611bbe7d5152, entries=150, sequenceid=74, filesize=11.7 K 2024-11-25T12:04:44,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/13d45ce6665d4cda93781c97052b6027 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/13d45ce6665d4cda93781c97052b6027 2024-11-25T12:04:44,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:44,242 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/13d45ce6665d4cda93781c97052b6027, entries=150, sequenceid=74, filesize=11.7 K 2024-11-25T12:04:44,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/263db9617fe240ea98175161e49ad131 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/263db9617fe240ea98175161e49ad131 2024-11-25T12:04:44,250 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/263db9617fe240ea98175161e49ad131, entries=150, sequenceid=74, filesize=11.7 K 2024-11-25T12:04:44,252 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 5f0aa52f88963b28a40fb54be46c7463 in 1143ms, sequenceid=74, compaction requested=false 2024-11-25T12:04:44,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:44,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:44,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-25T12:04:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-25T12:04:44,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-25T12:04:44,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3120 sec 2024-11-25T12:04:44,289 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 1.3500 sec 2024-11-25T12:04:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:44,707 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:44,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-25T12:04:44,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/fd67103b0b7c45e6ad4ec954251c0fa3 is 50, key is test_row_0/A:col10/1732536283528/Put/seqid=0 2024-11-25T12:04:44,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742065_1241 (size=12001) 2024-11-25T12:04:44,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/fd67103b0b7c45e6ad4ec954251c0fa3 2024-11-25T12:04:44,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/9599c4abb71446e489282a086671f8cf is 50, key is test_row_0/B:col10/1732536283528/Put/seqid=0 2024-11-25T12:04:44,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742066_1242 (size=12001) 2024-11-25T12:04:44,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536344807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:44,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:44,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536344815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:44,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:44,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:44,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536344816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:44,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:44,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536344818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:44,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:44,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536344926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:44,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:44,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536344929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:44,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:44,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536344936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:44,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:44,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:44,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536344941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:44,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:44,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:44,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47622 deadline: 1732536344989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:44,993 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_1' in 5f0aa52f88963b28a40fb54be46c7463 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:04:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:44,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-25T12:04:45,109 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:45,110 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:04:45,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-25T12:04:45,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T12:04:45,125 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:04:45,126 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:04:45,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:04:45,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536345136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:45,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:45,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:45,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536345136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:45,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:45,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:45,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:45,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/9599c4abb71446e489282a086671f8cf 2024-11-25T12:04:45,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536345150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:45,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536345156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:45,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/8accc138fad14d0ea9daeaabfe797296 is 50, key is test_row_0/C:col10/1732536283528/Put/seqid=0 2024-11-25T12:04:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T12:04:45,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742067_1243 (size=12001) 2024-11-25T12:04:45,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-25T12:04:45,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:45,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:45,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:45,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:45,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T12:04:45,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-25T12:04:45,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:45,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:45,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:45,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:45,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:45,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:45,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536345447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:45,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536345449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:45,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:45,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:45,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536345469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:45,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:45,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:45,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536345473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:45,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:45,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-25T12:04:45,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:45,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:45,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:45,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:45,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:45,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/8accc138fad14d0ea9daeaabfe797296 2024-11-25T12:04:45,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/fd67103b0b7c45e6ad4ec954251c0fa3 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/fd67103b0b7c45e6ad4ec954251c0fa3 2024-11-25T12:04:45,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/fd67103b0b7c45e6ad4ec954251c0fa3, entries=150, sequenceid=90, filesize=11.7 K 2024-11-25T12:04:45,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/9599c4abb71446e489282a086671f8cf as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/9599c4abb71446e489282a086671f8cf 2024-11-25T12:04:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T12:04:45,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-25T12:04:45,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:45,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:45,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:45,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:45,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:45,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/9599c4abb71446e489282a086671f8cf, entries=150, sequenceid=90, filesize=11.7 K 2024-11-25T12:04:45,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/8accc138fad14d0ea9daeaabfe797296 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/8accc138fad14d0ea9daeaabfe797296 2024-11-25T12:04:45,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/8accc138fad14d0ea9daeaabfe797296, entries=150, sequenceid=90, filesize=11.7 K 2024-11-25T12:04:45,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 5f0aa52f88963b28a40fb54be46c7463 in 1161ms, sequenceid=90, compaction requested=true 2024-11-25T12:04:45,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:45,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:04:45,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:45,869 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:45,869 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:45,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:04:45,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:45,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:04:45,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:45,890 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:45,890 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/A is initiating minor compaction (all files) 2024-11-25T12:04:45,890 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/A in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:45,890 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/712e028a104a43a6ab5d04cd6f9a36eb, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/5dc7e23630a3495b8466611bbe7d5152, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/fd67103b0b7c45e6ad4ec954251c0fa3] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=35.3 K 2024-11-25T12:04:45,890 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:45,890 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/B is initiating minor compaction (all files) 2024-11-25T12:04:45,890 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/B in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:45,891 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/6885fae639854f0190dfee976e8ec6a8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/13d45ce6665d4cda93781c97052b6027, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/9599c4abb71446e489282a086671f8cf] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=35.3 K 2024-11-25T12:04:45,897 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 712e028a104a43a6ab5d04cd6f9a36eb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732536281941 2024-11-25T12:04:45,897 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 6885fae639854f0190dfee976e8ec6a8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732536281941 2024-11-25T12:04:45,899 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dc7e23630a3495b8466611bbe7d5152, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732536282334 2024-11-25T12:04:45,899 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 13d45ce6665d4cda93781c97052b6027, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732536282334 2024-11-25T12:04:45,901 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd67103b0b7c45e6ad4ec954251c0fa3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732536283527 2024-11-25T12:04:45,905 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 9599c4abb71446e489282a086671f8cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732536283527 2024-11-25T12:04:45,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-25T12:04:45,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:45,929 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T12:04:45,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/e8bbbc896df642548ec925c06fd98676 is 50, key is test_row_0/A:col10/1732536284812/Put/seqid=0 2024-11-25T12:04:45,946 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#A#compaction#199 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:45,946 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/e8eb4892a4b44e8fb5dd688bee818fbf is 50, key is test_row_0/A:col10/1732536283528/Put/seqid=0 2024-11-25T12:04:45,951 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#B#compaction#200 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:45,951 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/db27b13b7fbf44208a4978d16fabf852 is 50, key is test_row_0/B:col10/1732536283528/Put/seqid=0 2024-11-25T12:04:45,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:46,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742068_1244 (size=12001) 2024-11-25T12:04:46,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742069_1245 (size=12207) 2024-11-25T12:04:46,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742070_1246 (size=12207) 2024-11-25T12:04:46,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536346069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:46,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:46,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536346075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:46,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:46,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536346088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:46,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:46,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536346092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:46,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:46,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:46,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536346199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:46,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:46,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536346203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:46,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:46,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536346207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:46,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536346207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:46,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:46,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T12:04:46,434 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/e8bbbc896df642548ec925c06fd98676 2024-11-25T12:04:46,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536346433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:46,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:46,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536346433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:46,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536346434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536346433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:46,500 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/e8eb4892a4b44e8fb5dd688bee818fbf as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/e8eb4892a4b44e8fb5dd688bee818fbf 2024-11-25T12:04:46,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/f007806087fc4e83b6ac6e54d66129f6 is 50, key is test_row_0/B:col10/1732536284812/Put/seqid=0 2024-11-25T12:04:46,513 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/db27b13b7fbf44208a4978d16fabf852 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/db27b13b7fbf44208a4978d16fabf852 2024-11-25T12:04:46,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742071_1247 (size=12001) 2024-11-25T12:04:46,553 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/A of 5f0aa52f88963b28a40fb54be46c7463 into e8eb4892a4b44e8fb5dd688bee818fbf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:46,553 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/B of 5f0aa52f88963b28a40fb54be46c7463 into db27b13b7fbf44208a4978d16fabf852(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:46,553 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:46,553 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:46,553 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/A, priority=13, startTime=1732536285869; duration=0sec 2024-11-25T12:04:46,553 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/B, priority=13, startTime=1732536285869; duration=0sec 2024-11-25T12:04:46,553 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:46,553 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:B 2024-11-25T12:04:46,554 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:46,560 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:46,560 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:A 2024-11-25T12:04:46,561 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:46,562 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/C is initiating minor compaction (all files) 2024-11-25T12:04:46,562 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/C in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:46,562 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/c2b7160087aa471296964e42aba06125, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/263db9617fe240ea98175161e49ad131, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/8accc138fad14d0ea9daeaabfe797296] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=35.3 K 2024-11-25T12:04:46,562 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting c2b7160087aa471296964e42aba06125, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732536281941 2024-11-25T12:04:46,563 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 263db9617fe240ea98175161e49ad131, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732536282334 2024-11-25T12:04:46,563 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 8accc138fad14d0ea9daeaabfe797296, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732536283527 2024-11-25T12:04:46,579 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#C#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:46,579 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/e8554aef716943099f54dea67ef0e201 is 50, key is test_row_0/C:col10/1732536283528/Put/seqid=0 2024-11-25T12:04:46,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742072_1248 (size=12207) 2024-11-25T12:04:46,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536346748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:46,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:46,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536346757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:46,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536346757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536346757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:46,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:46,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:46,956 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/f007806087fc4e83b6ac6e54d66129f6 2024-11-25T12:04:47,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/ae69f317e2094cab8572482b1e9d4570 is 50, key is test_row_0/C:col10/1732536284812/Put/seqid=0 2024-11-25T12:04:47,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742073_1249 (size=12001) 2024-11-25T12:04:47,076 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/ae69f317e2094cab8572482b1e9d4570 2024-11-25T12:04:47,088 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/e8554aef716943099f54dea67ef0e201 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/e8554aef716943099f54dea67ef0e201 2024-11-25T12:04:47,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/e8bbbc896df642548ec925c06fd98676 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/e8bbbc896df642548ec925c06fd98676 2024-11-25T12:04:47,121 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/C of 5f0aa52f88963b28a40fb54be46c7463 into e8554aef716943099f54dea67ef0e201(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:47,122 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:47,122 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/C, priority=13, startTime=1732536285869; duration=0sec 2024-11-25T12:04:47,122 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:47,122 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:C 2024-11-25T12:04:47,134 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/e8bbbc896df642548ec925c06fd98676, entries=150, sequenceid=114, filesize=11.7 K 2024-11-25T12:04:47,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/f007806087fc4e83b6ac6e54d66129f6 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/f007806087fc4e83b6ac6e54d66129f6 2024-11-25T12:04:47,166 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/f007806087fc4e83b6ac6e54d66129f6, entries=150, sequenceid=114, filesize=11.7 K 2024-11-25T12:04:47,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/ae69f317e2094cab8572482b1e9d4570 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ae69f317e2094cab8572482b1e9d4570 2024-11-25T12:04:47,212 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ae69f317e2094cab8572482b1e9d4570, entries=150, sequenceid=114, filesize=11.7 K 2024-11-25T12:04:47,222 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 5f0aa52f88963b28a40fb54be46c7463 in 1293ms, sequenceid=114, compaction requested=false 2024-11-25T12:04:47,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:47,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:47,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-25T12:04:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-25T12:04:47,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-25T12:04:47,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1080 sec 2024-11-25T12:04:47,239 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.1270 sec 2024-11-25T12:04:47,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-25T12:04:47,273 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:47,277 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:04:47,278 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:47,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-25T12:04:47,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T12:04:47,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T12:04:47,289 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:04:47,291 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:04:47,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:04:47,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/522aabe58b82486683ab26672b355673 is 50, key is test_row_0/A:col10/1732536287274/Put/seqid=0 2024-11-25T12:04:47,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742074_1250 (size=12101) 2024-11-25T12:04:47,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:47,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536347372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:47,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536347374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:47,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536347374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:47,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536347376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T12:04:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:47,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-25T12:04:47,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:47,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:47,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:47,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:47,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:47,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536347492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:47,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536347497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:47,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536347498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:47,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:47,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536347498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:47,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:47,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:47,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T12:04:47,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-25T12:04:47,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:47,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:47,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:47,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:47,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:47,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536347707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:47,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536347717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:47,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536347717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:47,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536347716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:47,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:47,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:47,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/522aabe58b82486683ab26672b355673 2024-11-25T12:04:47,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-25T12:04:47,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:47,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:47,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:47,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:47,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:47,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/6ba9a1e3cb8643cb813b7b2ddb7cc8e0 is 50, key is test_row_0/B:col10/1732536287274/Put/seqid=0 2024-11-25T12:04:47,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742075_1251 (size=12101) 2024-11-25T12:04:47,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/6ba9a1e3cb8643cb813b7b2ddb7cc8e0 2024-11-25T12:04:47,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/492433cccfbb4a479e891dec45e909b2 is 50, key is test_row_0/C:col10/1732536287274/Put/seqid=0 2024-11-25T12:04:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T12:04:47,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742076_1252 (size=12101) 2024-11-25T12:04:47,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-25T12:04:47,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:47,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:47,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:47,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:47,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:48,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:48,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536348016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:48,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:48,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:48,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536348038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:48,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:48,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:48,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536348039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:48,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536348038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:48,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:48,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:48,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:48,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-25T12:04:48,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:48,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:48,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:48,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:48,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:48,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-25T12:04:48,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:48,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:48,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:48,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:48,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:48,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/492433cccfbb4a479e891dec45e909b2 2024-11-25T12:04:48,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/522aabe58b82486683ab26672b355673 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/522aabe58b82486683ab26672b355673 2024-11-25T12:04:48,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/522aabe58b82486683ab26672b355673, entries=150, sequenceid=132, filesize=11.8 K 2024-11-25T12:04:48,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/6ba9a1e3cb8643cb813b7b2ddb7cc8e0 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/6ba9a1e3cb8643cb813b7b2ddb7cc8e0 2024-11-25T12:04:48,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/6ba9a1e3cb8643cb813b7b2ddb7cc8e0, entries=150, sequenceid=132, filesize=11.8 K 2024-11-25T12:04:48,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/492433cccfbb4a479e891dec45e909b2 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/492433cccfbb4a479e891dec45e909b2 2024-11-25T12:04:48,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/492433cccfbb4a479e891dec45e909b2, entries=150, sequenceid=132, filesize=11.8 K 2024-11-25T12:04:48,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 5f0aa52f88963b28a40fb54be46c7463 in 1113ms, sequenceid=132, compaction requested=true 2024-11-25T12:04:48,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:48,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:04:48,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:48,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:04:48,393 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:48,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:48,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:04:48,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:04:48,396 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:48,404 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:48,404 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:48,405 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/B is initiating minor compaction (all files) 2024-11-25T12:04:48,405 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/A is initiating minor compaction (all files) 2024-11-25T12:04:48,405 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/B in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:48,405 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/A in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:48,405 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/db27b13b7fbf44208a4978d16fabf852, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/f007806087fc4e83b6ac6e54d66129f6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/6ba9a1e3cb8643cb813b7b2ddb7cc8e0] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=35.5 K 2024-11-25T12:04:48,405 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting db27b13b7fbf44208a4978d16fabf852, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732536283527 2024-11-25T12:04:48,406 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/e8eb4892a4b44e8fb5dd688bee818fbf, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/e8bbbc896df642548ec925c06fd98676, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/522aabe58b82486683ab26672b355673] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=35.5 K 2024-11-25T12:04:48,406 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting f007806087fc4e83b6ac6e54d66129f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732536284812 2024-11-25T12:04:48,406 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting e8eb4892a4b44e8fb5dd688bee818fbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732536283527 2024-11-25T12:04:48,408 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ba9a1e3cb8643cb813b7b2ddb7cc8e0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536286079 2024-11-25T12:04:48,410 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting e8bbbc896df642548ec925c06fd98676, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732536284812 2024-11-25T12:04:48,415 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 522aabe58b82486683ab26672b355673, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536286079 2024-11-25T12:04:48,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-25T12:04:48,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:48,425 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-25T12:04:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T12:04:48,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/8af1c9752b8b4be29e4147a0794eacd5 is 50, key is test_row_0/A:col10/1732536287371/Put/seqid=0 2024-11-25T12:04:48,455 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#B#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:48,456 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/e296abe7f03a4530a2ba435700d7c753 is 50, key is test_row_0/B:col10/1732536287274/Put/seqid=0 2024-11-25T12:04:48,503 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#A#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:48,503 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/cdd50de6484344e8b45fc7214ff86328 is 50, key is test_row_0/A:col10/1732536287274/Put/seqid=0 2024-11-25T12:04:48,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742077_1253 (size=12151) 2024-11-25T12:04:48,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742078_1254 (size=12409) 2024-11-25T12:04:48,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:48,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742079_1255 (size=12409) 2024-11-25T12:04:48,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536348592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:48,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:48,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536348593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:48,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:48,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536348594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:48,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536348596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:48,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:48,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:48,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:48,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536348716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:48,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536348716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:48,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:48,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536348716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:48,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536348718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:48,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:48,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536348927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:48,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:48,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536348927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:48,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:48,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536348928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:48,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536348930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:48,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,937 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/8af1c9752b8b4be29e4147a0794eacd5 2024-11-25T12:04:48,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:48,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:48,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/d96eb73acdf048ac862f64e1251d2612 is 50, key is test_row_0/B:col10/1732536287371/Put/seqid=0 2024-11-25T12:04:48,974 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/e296abe7f03a4530a2ba435700d7c753 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/e296abe7f03a4530a2ba435700d7c753 2024-11-25T12:04:48,997 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/cdd50de6484344e8b45fc7214ff86328 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/cdd50de6484344e8b45fc7214ff86328 2024-11-25T12:04:49,001 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/B of 5f0aa52f88963b28a40fb54be46c7463 into e296abe7f03a4530a2ba435700d7c753(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:49,001 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:49,001 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/B, priority=13, startTime=1732536288393; duration=0sec 2024-11-25T12:04:49,001 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:49,001 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:B 2024-11-25T12:04:49,001 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:49,003 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:49,003 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/C is initiating minor compaction (all files) 2024-11-25T12:04:49,004 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/C in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:49,004 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/e8554aef716943099f54dea67ef0e201, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ae69f317e2094cab8572482b1e9d4570, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/492433cccfbb4a479e891dec45e909b2] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=35.5 K 2024-11-25T12:04:49,006 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8554aef716943099f54dea67ef0e201, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732536283527 2024-11-25T12:04:49,013 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae69f317e2094cab8572482b1e9d4570, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732536284812 2024-11-25T12:04:49,017 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 492433cccfbb4a479e891dec45e909b2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536286079 2024-11-25T12:04:49,024 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/A of 5f0aa52f88963b28a40fb54be46c7463 into cdd50de6484344e8b45fc7214ff86328(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:49,024 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:49,024 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/A, priority=13, startTime=1732536288393; duration=0sec 2024-11-25T12:04:49,025 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:49,025 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:A 2024-11-25T12:04:49,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742080_1256 (size=12151) 2024-11-25T12:04:49,038 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/d96eb73acdf048ac862f64e1251d2612 2024-11-25T12:04:49,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:49,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47622 deadline: 1732536349041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:49,049 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_1' in 5f0aa52f88963b28a40fb54be46c7463 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8276 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:04:49,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:49,058 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#C#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:49,059 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/ddcca5cd9972453e8e6a946588507bb5 is 50, key is test_row_0/C:col10/1732536287274/Put/seqid=0 2024-11-25T12:04:49,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/ca6a8b7605474f20ac4c98dcd76143be is 50, key is test_row_0/C:col10/1732536287371/Put/seqid=0 2024-11-25T12:04:49,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742081_1257 (size=12409) 2024-11-25T12:04:49,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742082_1258 (size=12151) 2024-11-25T12:04:49,140 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/ca6a8b7605474f20ac4c98dcd76143be 2024-11-25T12:04:49,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/8af1c9752b8b4be29e4147a0794eacd5 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/8af1c9752b8b4be29e4147a0794eacd5 2024-11-25T12:04:49,182 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/8af1c9752b8b4be29e4147a0794eacd5, entries=150, sequenceid=153, filesize=11.9 K 2024-11-25T12:04:49,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/d96eb73acdf048ac862f64e1251d2612 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/d96eb73acdf048ac862f64e1251d2612 2024-11-25T12:04:49,197 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/d96eb73acdf048ac862f64e1251d2612, entries=150, sequenceid=153, filesize=11.9 K 2024-11-25T12:04:49,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/ca6a8b7605474f20ac4c98dcd76143be as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ca6a8b7605474f20ac4c98dcd76143be 2024-11-25T12:04:49,211 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ca6a8b7605474f20ac4c98dcd76143be, entries=150, sequenceid=153, filesize=11.9 K 2024-11-25T12:04:49,212 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 5f0aa52f88963b28a40fb54be46c7463 in 787ms, sequenceid=153, compaction requested=false 2024-11-25T12:04:49,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:49,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:49,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-25T12:04:49,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-25T12:04:49,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-25T12:04:49,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9270 sec 2024-11-25T12:04:49,223 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.9450 sec 2024-11-25T12:04:49,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:49,251 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:49,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-25T12:04:49,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/7ba15caeb23644e688df33c9d287143f is 50, key is test_row_0/A:col10/1732536288558/Put/seqid=0 2024-11-25T12:04:49,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742083_1259 (size=12151) 2024-11-25T12:04:49,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:49,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536349356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:49,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:49,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536349356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:49,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:49,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536349357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:49,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:49,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536349359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:49,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-25T12:04:49,456 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:49,457 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:04:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-25T12:04:49,466 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:04:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T12:04:49,466 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:04:49,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:04:49,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536349467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:49,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536349467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:49,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536349468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:49,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:49,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536349478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:49,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:49,523 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/ddcca5cd9972453e8e6a946588507bb5 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ddcca5cd9972453e8e6a946588507bb5 2024-11-25T12:04:49,530 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/C of 5f0aa52f88963b28a40fb54be46c7463 into ddcca5cd9972453e8e6a946588507bb5(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:49,530 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:49,530 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/C, priority=13, startTime=1732536288393; duration=0sec 2024-11-25T12:04:49,530 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:49,530 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:C 2024-11-25T12:04:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T12:04:49,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T12:04:49,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:49,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:49,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:49,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:49,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:49,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536349687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:49,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:49,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536349687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:49,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:49,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536349687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:49,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:49,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536349697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:49,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:49,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:49,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/7ba15caeb23644e688df33c9d287143f 2024-11-25T12:04:49,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/c876f4295a124087847a4b80b870079f is 50, key is test_row_0/B:col10/1732536288558/Put/seqid=0 2024-11-25T12:04:49,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T12:04:49,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:49,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:49,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:49,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:49,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T12:04:49,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:49,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742084_1260 (size=12151) 2024-11-25T12:04:49,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/c876f4295a124087847a4b80b870079f 2024-11-25T12:04:49,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/f4c2cc027f694d428646d97bbad0bb22 is 50, key is test_row_0/C:col10/1732536288558/Put/seqid=0 2024-11-25T12:04:49,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742085_1261 (size=12151) 2024-11-25T12:04:49,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T12:04:49,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:49,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:49,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:49,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:49,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:50,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536349997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:50,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:50,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536349997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:50,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536350004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:50,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:50,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:50,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536350006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:50,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:50,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T12:04:50,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T12:04:50,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:50,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:50,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:50,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:50,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:50,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T12:04:50,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:50,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:50,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:50,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:50,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:50,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:50,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/f4c2cc027f694d428646d97bbad0bb22 2024-11-25T12:04:50,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/7ba15caeb23644e688df33c9d287143f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/7ba15caeb23644e688df33c9d287143f 2024-11-25T12:04:50,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/7ba15caeb23644e688df33c9d287143f, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T12:04:50,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/c876f4295a124087847a4b80b870079f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/c876f4295a124087847a4b80b870079f 2024-11-25T12:04:50,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/c876f4295a124087847a4b80b870079f, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T12:04:50,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/f4c2cc027f694d428646d97bbad0bb22 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/f4c2cc027f694d428646d97bbad0bb22 2024-11-25T12:04:50,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/f4c2cc027f694d428646d97bbad0bb22, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T12:04:50,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 5f0aa52f88963b28a40fb54be46c7463 in 1138ms, sequenceid=172, compaction requested=true 2024-11-25T12:04:50,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:50,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:04:50,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:50,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:04:50,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:04:50,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:04:50,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:04:50,390 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:50,390 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:50,393 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:50,393 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/A is initiating minor compaction (all files) 2024-11-25T12:04:50,393 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/A in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:50,394 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/cdd50de6484344e8b45fc7214ff86328, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/8af1c9752b8b4be29e4147a0794eacd5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/7ba15caeb23644e688df33c9d287143f] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=35.9 K 2024-11-25T12:04:50,394 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting cdd50de6484344e8b45fc7214ff86328, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536286079 2024-11-25T12:04:50,395 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 8af1c9752b8b4be29e4147a0794eacd5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732536287370 2024-11-25T12:04:50,397 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ba15caeb23644e688df33c9d287143f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732536288558 2024-11-25T12:04:50,404 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:50,405 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/C is initiating minor compaction (all files) 2024-11-25T12:04:50,405 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/C in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:50,405 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ddcca5cd9972453e8e6a946588507bb5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ca6a8b7605474f20ac4c98dcd76143be, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/f4c2cc027f694d428646d97bbad0bb22] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=35.9 K 2024-11-25T12:04:50,408 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddcca5cd9972453e8e6a946588507bb5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536286079 2024-11-25T12:04:50,408 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca6a8b7605474f20ac4c98dcd76143be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732536287370 2024-11-25T12:04:50,414 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4c2cc027f694d428646d97bbad0bb22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732536288558 2024-11-25T12:04:50,425 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#A#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:50,426 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/7bb3767e0d3f48f9a6284fe80491062a is 50, key is test_row_0/A:col10/1732536288558/Put/seqid=0 2024-11-25T12:04:50,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-25T12:04:50,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:50,429 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-25T12:04:50,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/17177deb3f2b4242a41fb6537eb4d33d is 50, key is test_row_0/A:col10/1732536289353/Put/seqid=0 2024-11-25T12:04:50,475 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#C#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:50,476 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/a3960e0c40e64fdc8bea6bd18d77465c is 50, key is test_row_0/C:col10/1732536288558/Put/seqid=0 2024-11-25T12:04:50,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742086_1262 (size=12561) 2024-11-25T12:04:50,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:50,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:50,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742087_1263 (size=12151) 2024-11-25T12:04:50,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742088_1264 (size=12561) 2024-11-25T12:04:50,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:50,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536350557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:50,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:50,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536350560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:50,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:50,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536350561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:50,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536350562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T12:04:50,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536350677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:50,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:50,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:50,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536350678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:50,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536350680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:50,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536350680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:50,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:50,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536350886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:50,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:50,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536350886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:50,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:50,901 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/7bb3767e0d3f48f9a6284fe80491062a as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/7bb3767e0d3f48f9a6284fe80491062a 2024-11-25T12:04:50,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536350897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:50,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536350897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:50,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:50,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:50,927 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/A of 5f0aa52f88963b28a40fb54be46c7463 into 7bb3767e0d3f48f9a6284fe80491062a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:50,928 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:50,928 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/A, priority=13, startTime=1732536290389; duration=0sec 2024-11-25T12:04:50,928 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:50,928 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:A 2024-11-25T12:04:50,928 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:50,929 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/17177deb3f2b4242a41fb6537eb4d33d 2024-11-25T12:04:50,937 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:50,937 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/B is initiating minor compaction (all files) 2024-11-25T12:04:50,937 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/B in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:50,938 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/e296abe7f03a4530a2ba435700d7c753, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/d96eb73acdf048ac862f64e1251d2612, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/c876f4295a124087847a4b80b870079f] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=35.9 K 2024-11-25T12:04:50,940 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting e296abe7f03a4530a2ba435700d7c753, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536286079 2024-11-25T12:04:50,941 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting d96eb73acdf048ac862f64e1251d2612, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732536287370 2024-11-25T12:04:50,942 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting c876f4295a124087847a4b80b870079f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732536288558 2024-11-25T12:04:50,955 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/a3960e0c40e64fdc8bea6bd18d77465c as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/a3960e0c40e64fdc8bea6bd18d77465c 2024-11-25T12:04:50,962 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/C of 5f0aa52f88963b28a40fb54be46c7463 into a3960e0c40e64fdc8bea6bd18d77465c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:50,962 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:50,962 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/C, priority=13, startTime=1732536290389; duration=0sec 2024-11-25T12:04:50,962 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:50,962 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:C 2024-11-25T12:04:50,971 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#B#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:50,971 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/fb12173e3d5c465fb9af9b4e3e11fc21 is 50, key is test_row_0/B:col10/1732536288558/Put/seqid=0 2024-11-25T12:04:50,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/3121ff1583e045a6a1a6dc38de2f1959 is 50, key is test_row_0/B:col10/1732536289353/Put/seqid=0 2024-11-25T12:04:51,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742089_1265 (size=12561) 2024-11-25T12:04:51,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742090_1266 (size=12151) 2024-11-25T12:04:51,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536351201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:51,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:51,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:51,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:51,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536351207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:51,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536351218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:51,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536351218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:51,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:51,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:51,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:51,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:51,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:51,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:51,414 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/3121ff1583e045a6a1a6dc38de2f1959 2024-11-25T12:04:51,428 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/fb12173e3d5c465fb9af9b4e3e11fc21 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/fb12173e3d5c465fb9af9b4e3e11fc21 2024-11-25T12:04:51,465 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/B of 5f0aa52f88963b28a40fb54be46c7463 into fb12173e3d5c465fb9af9b4e3e11fc21(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:51,465 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:51,465 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/B, priority=13, startTime=1732536290389; duration=0sec 2024-11-25T12:04:51,465 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:51,465 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:B 2024-11-25T12:04:51,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/adea093cd426441fa69fc2e4c599e698 is 50, key is test_row_0/C:col10/1732536289353/Put/seqid=0 2024-11-25T12:04:51,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742091_1267 (size=12151) 2024-11-25T12:04:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T12:04:51,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536351722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:51,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:51,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:51,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:51,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536351727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:51,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:51,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536351736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:51,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536351736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:51,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:51,922 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/adea093cd426441fa69fc2e4c599e698 2024-11-25T12:04:51,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/17177deb3f2b4242a41fb6537eb4d33d as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/17177deb3f2b4242a41fb6537eb4d33d 2024-11-25T12:04:52,004 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/17177deb3f2b4242a41fb6537eb4d33d, entries=150, sequenceid=192, filesize=11.9 K 2024-11-25T12:04:52,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/3121ff1583e045a6a1a6dc38de2f1959 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/3121ff1583e045a6a1a6dc38de2f1959 2024-11-25T12:04:52,053 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/3121ff1583e045a6a1a6dc38de2f1959, entries=150, sequenceid=192, filesize=11.9 K 2024-11-25T12:04:52,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/adea093cd426441fa69fc2e4c599e698 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/adea093cd426441fa69fc2e4c599e698 2024-11-25T12:04:52,068 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/adea093cd426441fa69fc2e4c599e698, entries=150, sequenceid=192, filesize=11.9 K 2024-11-25T12:04:52,070 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 5f0aa52f88963b28a40fb54be46c7463 in 1641ms, sequenceid=192, compaction requested=false 2024-11-25T12:04:52,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:52,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:52,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-25T12:04:52,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-25T12:04:52,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-25T12:04:52,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6130 sec 2024-11-25T12:04:52,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.6370 sec 2024-11-25T12:04:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:52,748 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:52,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-25T12:04:52,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/745fad038e2348f8947db2fb36d546e5 is 50, key is test_row_0/A:col10/1732536290553/Put/seqid=0 2024-11-25T12:04:52,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742092_1268 (size=14541) 2024-11-25T12:04:52,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/745fad038e2348f8947db2fb36d546e5 2024-11-25T12:04:52,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:52,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536352814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:52,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:52,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536352814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:52,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536352820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:52,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:52,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:52,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:52,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536352821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:52,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/21d7f74d77ad40138528827c5a6b9f1f is 50, key is test_row_0/B:col10/1732536290553/Put/seqid=0 2024-11-25T12:04:52,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742093_1269 (size=12151) 2024-11-25T12:04:52,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536352936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:52,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536352936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:52,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536352937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:52,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536352936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:52,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:52,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:52,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:52,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:52,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:52,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:53,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:53,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536353148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:53,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:53,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536353150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:53,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536353157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:53,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:53,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:53,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:53,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:53,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536353160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:53,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:53,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/21d7f74d77ad40138528827c5a6b9f1f 2024-11-25T12:04:53,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/88067c9030ac4f38ae5925b8e94bc2ea is 50, key is test_row_0/C:col10/1732536290553/Put/seqid=0 2024-11-25T12:04:53,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742094_1270 (size=12151) 2024-11-25T12:04:53,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:53,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536353466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:53,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:53,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536353466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:53,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:53,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536353477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:53,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:53,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536353481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:53,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:53,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:53,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:53,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-25T12:04:53,657 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:53,658 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:04:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-25T12:04:53,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T12:04:53,660 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:04:53,661 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:04:53,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:04:53,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T12:04:53,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/88067c9030ac4f38ae5925b8e94bc2ea 2024-11-25T12:04:53,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/745fad038e2348f8947db2fb36d546e5 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/745fad038e2348f8947db2fb36d546e5 2024-11-25T12:04:53,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/745fad038e2348f8947db2fb36d546e5, entries=200, sequenceid=214, filesize=14.2 K 2024-11-25T12:04:53,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/21d7f74d77ad40138528827c5a6b9f1f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/21d7f74d77ad40138528827c5a6b9f1f 2024-11-25T12:04:53,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/21d7f74d77ad40138528827c5a6b9f1f, entries=150, sequenceid=214, filesize=11.9 K 2024-11-25T12:04:53,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/88067c9030ac4f38ae5925b8e94bc2ea as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/88067c9030ac4f38ae5925b8e94bc2ea 2024-11-25T12:04:53,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T12:04:53,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:53,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:53,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:53,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:53,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:53,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/88067c9030ac4f38ae5925b8e94bc2ea, entries=150, sequenceid=214, filesize=11.9 K 2024-11-25T12:04:53,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 5f0aa52f88963b28a40fb54be46c7463 in 1117ms, sequenceid=214, compaction requested=true 2024-11-25T12:04:53,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:53,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:04:53,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:53,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:04:53,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:53,865 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:53,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:04:53,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:04:53,865 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:53,871 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:53,871 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/B is initiating minor compaction (all files) 2024-11-25T12:04:53,871 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/B in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:53,872 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:53,872 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/A is initiating minor compaction (all files) 2024-11-25T12:04:53,872 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/A in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:53,872 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/7bb3767e0d3f48f9a6284fe80491062a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/17177deb3f2b4242a41fb6537eb4d33d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/745fad038e2348f8947db2fb36d546e5] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=38.3 K 2024-11-25T12:04:53,872 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/fb12173e3d5c465fb9af9b4e3e11fc21, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/3121ff1583e045a6a1a6dc38de2f1959, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/21d7f74d77ad40138528827c5a6b9f1f] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=36.0 K 2024-11-25T12:04:53,872 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bb3767e0d3f48f9a6284fe80491062a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732536288558 2024-11-25T12:04:53,874 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb12173e3d5c465fb9af9b4e3e11fc21, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732536288558 2024-11-25T12:04:53,874 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 17177deb3f2b4242a41fb6537eb4d33d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732536289348 2024-11-25T12:04:53,875 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3121ff1583e045a6a1a6dc38de2f1959, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732536289348 2024-11-25T12:04:53,875 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 745fad038e2348f8947db2fb36d546e5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732536290553 2024-11-25T12:04:53,875 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21d7f74d77ad40138528827c5a6b9f1f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732536290553 2024-11-25T12:04:53,899 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#A#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:53,900 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/202703593ece4de9bb3e1b9879de8c67 is 50, key is test_row_0/A:col10/1732536290553/Put/seqid=0 2024-11-25T12:04:53,902 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#B#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:53,903 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/b048ec2bc386406c9de526ce52f0ee50 is 50, key is test_row_0/B:col10/1732536290553/Put/seqid=0 2024-11-25T12:04:53,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742095_1271 (size=12663) 2024-11-25T12:04:53,965 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/202703593ece4de9bb3e1b9879de8c67 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/202703593ece4de9bb3e1b9879de8c67 2024-11-25T12:04:53,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742096_1272 (size=12663) 2024-11-25T12:04:53,972 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/A of 5f0aa52f88963b28a40fb54be46c7463 into 202703593ece4de9bb3e1b9879de8c67(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:53,972 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:53,972 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/A, priority=13, startTime=1732536293865; duration=0sec 2024-11-25T12:04:53,973 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:53,973 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:A 2024-11-25T12:04:53,973 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:53,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-25T12:04:53,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:53,973 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-25T12:04:53,976 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:53,976 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/C is initiating minor compaction (all files) 2024-11-25T12:04:53,976 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/C in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:53,976 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/a3960e0c40e64fdc8bea6bd18d77465c, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/adea093cd426441fa69fc2e4c599e698, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/88067c9030ac4f38ae5925b8e94bc2ea] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=36.0 K 2024-11-25T12:04:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T12:04:53,977 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting a3960e0c40e64fdc8bea6bd18d77465c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732536288558 2024-11-25T12:04:53,983 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting adea093cd426441fa69fc2e4c599e698, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732536289348 2024-11-25T12:04:53,984 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 88067c9030ac4f38ae5925b8e94bc2ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732536290553 2024-11-25T12:04:53,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/2db0c51700f049f1b34f95556c4cd2d6 is 50, key is test_row_0/A:col10/1732536292810/Put/seqid=0 2024-11-25T12:04:53,991 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/b048ec2bc386406c9de526ce52f0ee50 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/b048ec2bc386406c9de526ce52f0ee50 2024-11-25T12:04:53,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:54,000 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/B of 5f0aa52f88963b28a40fb54be46c7463 into b048ec2bc386406c9de526ce52f0ee50(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:54,001 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:54,001 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/B, priority=13, startTime=1732536293865; duration=0sec 2024-11-25T12:04:54,001 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:54,001 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:B 2024-11-25T12:04:54,005 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#C#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:54,006 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/a0a8edf57b6e40b5bfa6c371873f428a is 50, key is test_row_0/C:col10/1732536290553/Put/seqid=0 2024-11-25T12:04:54,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742097_1273 (size=12151) 2024-11-25T12:04:54,037 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/2db0c51700f049f1b34f95556c4cd2d6 2024-11-25T12:04:54,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742098_1274 (size=12663) 2024-11-25T12:04:54,066 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/a0a8edf57b6e40b5bfa6c371873f428a as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/a0a8edf57b6e40b5bfa6c371873f428a 2024-11-25T12:04:54,073 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/C of 5f0aa52f88963b28a40fb54be46c7463 into a0a8edf57b6e40b5bfa6c371873f428a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:54,073 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:54,073 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/C, priority=13, startTime=1732536293865; duration=0sec 2024-11-25T12:04:54,074 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:54,074 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:C 2024-11-25T12:04:54,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/3a9cdd2eebdd47298147f39992c143a6 is 50, key is test_row_0/B:col10/1732536292810/Put/seqid=0 2024-11-25T12:04:54,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:54,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536354069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:54,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:54,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536354073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:54,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536354077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:54,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:54,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536354089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:54,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742099_1275 (size=12151) 2024-11-25T12:04:54,127 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/3a9cdd2eebdd47298147f39992c143a6 2024-11-25T12:04:54,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/11c5ced748d94a558aedc7174b39261e is 50, key is test_row_0/C:col10/1732536292810/Put/seqid=0 2024-11-25T12:04:54,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742100_1276 (size=12151) 2024-11-25T12:04:54,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536354196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:54,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536354197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:54,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:54,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536354216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:54,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:54,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536354216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:54,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T12:04:54,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536354409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:54,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:54,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536354416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:54,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:54,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536354427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:54,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536354427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:54,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:54,568 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/11c5ced748d94a558aedc7174b39261e 2024-11-25T12:04:54,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/2db0c51700f049f1b34f95556c4cd2d6 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/2db0c51700f049f1b34f95556c4cd2d6 2024-11-25T12:04:54,587 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/2db0c51700f049f1b34f95556c4cd2d6, entries=150, sequenceid=232, filesize=11.9 K 2024-11-25T12:04:54,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/3a9cdd2eebdd47298147f39992c143a6 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/3a9cdd2eebdd47298147f39992c143a6 2024-11-25T12:04:54,602 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/3a9cdd2eebdd47298147f39992c143a6, entries=150, sequenceid=232, filesize=11.9 K 2024-11-25T12:04:54,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/11c5ced748d94a558aedc7174b39261e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/11c5ced748d94a558aedc7174b39261e 2024-11-25T12:04:54,623 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/11c5ced748d94a558aedc7174b39261e, entries=150, sequenceid=232, filesize=11.9 K 2024-11-25T12:04:54,626 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 5f0aa52f88963b28a40fb54be46c7463 in 653ms, sequenceid=232, compaction requested=false 2024-11-25T12:04:54,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:54,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:54,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-25T12:04:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-25T12:04:54,636 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-25T12:04:54,636 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 971 msec 2024-11-25T12:04:54,647 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 983 msec 2024-11-25T12:04:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:54,735 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:54,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-25T12:04:54,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/98f42c7e00514a6c97628deb6e71514f is 50, key is test_row_0/A:col10/1732536294729/Put/seqid=0 2024-11-25T12:04:54,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742101_1277 (size=12151) 2024-11-25T12:04:54,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/98f42c7e00514a6c97628deb6e71514f 2024-11-25T12:04:54,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/22c7c87341934cf7b646c4aa68c65ae3 is 50, key is test_row_0/B:col10/1732536294729/Put/seqid=0 2024-11-25T12:04:54,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536354790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:54,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536354790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:54,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-25T12:04:54,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:54,801 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:54,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536354792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:54,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:54,806 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:04:54,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536354798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-25T12:04:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T12:04:54,816 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:04:54,817 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:04:54,817 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:04:54,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742102_1278 (size=12151) 2024-11-25T12:04:54,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:54,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/22c7c87341934cf7b646c4aa68c65ae3 2024-11-25T12:04:54,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/ab92773241244c37a6e2f3b9b764fb75 is 50, key is test_row_0/C:col10/1732536294729/Put/seqid=0 2024-11-25T12:04:54,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742103_1279 (size=12151) 2024-11-25T12:04:54,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/ab92773241244c37a6e2f3b9b764fb75 2024-11-25T12:04:54,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536354906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:54,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536354906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:54,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536354906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T12:04:54,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536354934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:54,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/98f42c7e00514a6c97628deb6e71514f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/98f42c7e00514a6c97628deb6e71514f 2024-11-25T12:04:54,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/98f42c7e00514a6c97628deb6e71514f, entries=150, sequenceid=254, filesize=11.9 K 2024-11-25T12:04:54,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/22c7c87341934cf7b646c4aa68c65ae3 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/22c7c87341934cf7b646c4aa68c65ae3 2024-11-25T12:04:54,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T12:04:54,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:54,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:54,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:54,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:54,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:54,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/22c7c87341934cf7b646c4aa68c65ae3, entries=150, sequenceid=254, filesize=11.9 K 2024-11-25T12:04:54,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/ab92773241244c37a6e2f3b9b764fb75 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ab92773241244c37a6e2f3b9b764fb75 2024-11-25T12:04:55,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ab92773241244c37a6e2f3b9b764fb75, entries=150, sequenceid=254, filesize=11.9 K 2024-11-25T12:04:55,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 5f0aa52f88963b28a40fb54be46c7463 in 274ms, sequenceid=254, compaction requested=true 2024-11-25T12:04:55,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:55,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:04:55,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:55,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:04:55,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:04:55,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:04:55,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:04:55,009 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:55,010 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:55,014 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:55,014 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/A is initiating minor compaction (all files) 2024-11-25T12:04:55,014 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/A in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:55,014 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/202703593ece4de9bb3e1b9879de8c67, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/2db0c51700f049f1b34f95556c4cd2d6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/98f42c7e00514a6c97628deb6e71514f] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=36.1 K 2024-11-25T12:04:55,014 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 202703593ece4de9bb3e1b9879de8c67, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732536290553 2024-11-25T12:04:55,016 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 2db0c51700f049f1b34f95556c4cd2d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732536292810 2024-11-25T12:04:55,017 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:55,017 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/C is initiating minor compaction (all files) 2024-11-25T12:04:55,017 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/C in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:55,017 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/a0a8edf57b6e40b5bfa6c371873f428a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/11c5ced748d94a558aedc7174b39261e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ab92773241244c37a6e2f3b9b764fb75] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=36.1 K 2024-11-25T12:04:55,017 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 98f42c7e00514a6c97628deb6e71514f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732536294050 2024-11-25T12:04:55,018 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0a8edf57b6e40b5bfa6c371873f428a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732536290553 2024-11-25T12:04:55,019 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11c5ced748d94a558aedc7174b39261e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732536292810 2024-11-25T12:04:55,020 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab92773241244c37a6e2f3b9b764fb75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732536294050 2024-11-25T12:04:55,035 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#A#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:55,035 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/a6220b24307f4156aa04ac7628a2a43d is 50, key is test_row_0/A:col10/1732536294729/Put/seqid=0 2024-11-25T12:04:55,045 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#C#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:55,046 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/d0c14977d506454399f5b48d6c66d0ab is 50, key is test_row_0/C:col10/1732536294729/Put/seqid=0 2024-11-25T12:04:55,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742104_1280 (size=12765) 2024-11-25T12:04:55,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742105_1281 (size=12765) 2024-11-25T12:04:55,120 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/a6220b24307f4156aa04ac7628a2a43d as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/a6220b24307f4156aa04ac7628a2a43d 2024-11-25T12:04:55,125 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/d0c14977d506454399f5b48d6c66d0ab as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/d0c14977d506454399f5b48d6c66d0ab 2024-11-25T12:04:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:55,126 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:55,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-25T12:04:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T12:04:55,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T12:04:55,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:55,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:55,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:55,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:55,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:55,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/cde9599b50ed40d7946cc8e6996370bd is 50, key is test_row_0/A:col10/1732536295121/Put/seqid=0 2024-11-25T12:04:55,160 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/A of 5f0aa52f88963b28a40fb54be46c7463 into a6220b24307f4156aa04ac7628a2a43d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:55,160 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:55,160 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/A, priority=13, startTime=1732536295009; duration=0sec 2024-11-25T12:04:55,160 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:55,160 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:A 2024-11-25T12:04:55,160 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:55,161 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/C of 5f0aa52f88963b28a40fb54be46c7463 into d0c14977d506454399f5b48d6c66d0ab(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:55,161 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:55,161 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/C, priority=13, startTime=1732536295009; duration=0sec 2024-11-25T12:04:55,162 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:55,162 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:C 2024-11-25T12:04:55,165 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:55,165 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/B is initiating minor compaction (all files) 2024-11-25T12:04:55,165 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/B in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:55,165 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/b048ec2bc386406c9de526ce52f0ee50, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/3a9cdd2eebdd47298147f39992c143a6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/22c7c87341934cf7b646c4aa68c65ae3] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=36.1 K 2024-11-25T12:04:55,170 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting b048ec2bc386406c9de526ce52f0ee50, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732536290553 2024-11-25T12:04:55,173 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a9cdd2eebdd47298147f39992c143a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732536292810 2024-11-25T12:04:55,173 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 22c7c87341934cf7b646c4aa68c65ae3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732536294050 2024-11-25T12:04:55,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742106_1282 (size=14741) 2024-11-25T12:04:55,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/cde9599b50ed40d7946cc8e6996370bd 2024-11-25T12:04:55,209 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#B#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:55,210 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/9828ff69c1f445f6b5a446ca1e1a547a is 50, key is test_row_0/B:col10/1732536294729/Put/seqid=0 2024-11-25T12:04:55,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/d24856cf84c34016a276ed56cee9c2c2 is 50, key is test_row_0/B:col10/1732536295121/Put/seqid=0 2024-11-25T12:04:55,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:55,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536355203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:55,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:55,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536355221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:55,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:55,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536355224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:55,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:55,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536355233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:55,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742107_1283 (size=12765) 2024-11-25T12:04:55,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742108_1284 (size=12301) 2024-11-25T12:04:55,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/d24856cf84c34016a276ed56cee9c2c2 2024-11-25T12:04:55,291 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/9828ff69c1f445f6b5a446ca1e1a547a as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/9828ff69c1f445f6b5a446ca1e1a547a 2024-11-25T12:04:55,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T12:04:55,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:55,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:55,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:55,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:55,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:55,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:55,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/9a6c01548a9c4aa693c68a1256be6c37 is 50, key is test_row_0/C:col10/1732536295121/Put/seqid=0 2024-11-25T12:04:55,311 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/B of 5f0aa52f88963b28a40fb54be46c7463 into 9828ff69c1f445f6b5a446ca1e1a547a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:55,311 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:55,311 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/B, priority=13, startTime=1732536295009; duration=0sec 2024-11-25T12:04:55,311 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:55,312 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:B 2024-11-25T12:04:55,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536355327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:55,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742109_1285 (size=12301) 2024-11-25T12:04:55,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536355346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:55,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536355346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:55,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/9a6c01548a9c4aa693c68a1256be6c37 2024-11-25T12:04:55,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536355349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:55,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:55,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/cde9599b50ed40d7946cc8e6996370bd as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/cde9599b50ed40d7946cc8e6996370bd 2024-11-25T12:04:55,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/cde9599b50ed40d7946cc8e6996370bd, entries=200, sequenceid=271, filesize=14.4 K 2024-11-25T12:04:55,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/d24856cf84c34016a276ed56cee9c2c2 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/d24856cf84c34016a276ed56cee9c2c2 2024-11-25T12:04:55,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/d24856cf84c34016a276ed56cee9c2c2, entries=150, sequenceid=271, filesize=12.0 K 2024-11-25T12:04:55,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/9a6c01548a9c4aa693c68a1256be6c37 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/9a6c01548a9c4aa693c68a1256be6c37 2024-11-25T12:04:55,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/9a6c01548a9c4aa693c68a1256be6c37, entries=150, sequenceid=271, filesize=12.0 K 2024-11-25T12:04:55,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 5f0aa52f88963b28a40fb54be46c7463 in 283ms, sequenceid=271, compaction requested=false 2024-11-25T12:04:55,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:55,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T12:04:55,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-25T12:04:55,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:55,452 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-25T12:04:55,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/0190d74acd424bbc9c4a78d3b6da83a6 is 50, key is test_row_0/A:col10/1732536295220/Put/seqid=0 2024-11-25T12:04:55,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742110_1286 (size=12301) 2024-11-25T12:04:55,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:55,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:55,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536355613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536355616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:55,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536355617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:55,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536355617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:55,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:55,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:55,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:55,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:55,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536355736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:55,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536355736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:55,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536355736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:55,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536355736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:55,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:55,927 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/0190d74acd424bbc9c4a78d3b6da83a6 2024-11-25T12:04:55,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536355946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:55,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536355947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:55,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:55,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:55,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536355947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T12:04:55,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:55,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536355956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:55,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:55,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/9d70ad28c2b948d5bad1a12884e0cfe6 is 50, key is test_row_0/B:col10/1732536295220/Put/seqid=0 2024-11-25T12:04:56,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742111_1287 (size=12301) 2024-11-25T12:04:56,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536356256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:56,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:56,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536356256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:56,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:56,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:56,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536356267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:56,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536356267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:56,415 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/9d70ad28c2b948d5bad1a12884e0cfe6 2024-11-25T12:04:56,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/cc81c32e773c4725bd4fae25204ce459 is 50, key is test_row_0/C:col10/1732536295220/Put/seqid=0 2024-11-25T12:04:56,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742112_1288 (size=12301) 2024-11-25T12:04:56,462 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/cc81c32e773c4725bd4fae25204ce459 2024-11-25T12:04:56,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/0190d74acd424bbc9c4a78d3b6da83a6 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/0190d74acd424bbc9c4a78d3b6da83a6 2024-11-25T12:04:56,474 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/0190d74acd424bbc9c4a78d3b6da83a6, entries=150, sequenceid=293, filesize=12.0 K 2024-11-25T12:04:56,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/9d70ad28c2b948d5bad1a12884e0cfe6 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/9d70ad28c2b948d5bad1a12884e0cfe6 2024-11-25T12:04:56,501 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/9d70ad28c2b948d5bad1a12884e0cfe6, entries=150, sequenceid=293, filesize=12.0 K 2024-11-25T12:04:56,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/cc81c32e773c4725bd4fae25204ce459 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/cc81c32e773c4725bd4fae25204ce459 2024-11-25T12:04:56,511 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/cc81c32e773c4725bd4fae25204ce459, entries=150, sequenceid=293, filesize=12.0 K 2024-11-25T12:04:56,513 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 5f0aa52f88963b28a40fb54be46c7463 in 1060ms, sequenceid=293, compaction requested=true 2024-11-25T12:04:56,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:56,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:56,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-25T12:04:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-25T12:04:56,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-25T12:04:56,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6990 sec 2024-11-25T12:04:56,525 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.7170 sec 2024-11-25T12:04:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:56,774 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:56,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-25T12:04:56,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/a897c2a6c00e4a7f807fae7c0551e0a8 is 50, key is test_row_0/A:col10/1732536295612/Put/seqid=0 2024-11-25T12:04:56,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742113_1289 (size=12301) 2024-11-25T12:04:56,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/a897c2a6c00e4a7f807fae7c0551e0a8 2024-11-25T12:04:56,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536356819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:56,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:56,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536356825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:56,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,833 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:56,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:56,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536356828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:56,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:56,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536356828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:56,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:56,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/20dfc6957fb34927a74ba6fcc86fbc95 is 50, key is test_row_0/B:col10/1732536295612/Put/seqid=0 2024-11-25T12:04:56,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742114_1290 (size=12301) 2024-11-25T12:04:56,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536356936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:56,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536356936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:56,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:56,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536356949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:56,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:56,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536356953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:56,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:56,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:56,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-25T12:04:56,984 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:56,992 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:04:57,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-25T12:04:57,004 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:04:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T12:04:57,008 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:04:57,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:04:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T12:04:57,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536357148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:57,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536357156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:57,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T12:04:57,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:57,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:57,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:57,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:57,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:57,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536357166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:57,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536357166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:57,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:57,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:57,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:57,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:57,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:57,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:57,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/20dfc6957fb34927a74ba6fcc86fbc95 2024-11-25T12:04:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T12:04:57,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T12:04:57,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:57,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:57,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:57,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:57,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:57,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/2fb00e8003944c5c84d2273809091e55 is 50, key is test_row_0/C:col10/1732536295612/Put/seqid=0 2024-11-25T12:04:57,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742115_1291 (size=12301) 2024-11-25T12:04:57,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/2fb00e8003944c5c84d2273809091e55 2024-11-25T12:04:57,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/a897c2a6c00e4a7f807fae7c0551e0a8 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/a897c2a6c00e4a7f807fae7c0551e0a8 2024-11-25T12:04:57,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536357467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:57,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:57,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:57,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:57,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536357467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:57,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:57,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/a897c2a6c00e4a7f807fae7c0551e0a8, entries=150, sequenceid=311, filesize=12.0 K 2024-11-25T12:04:57,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/20dfc6957fb34927a74ba6fcc86fbc95 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/20dfc6957fb34927a74ba6fcc86fbc95 2024-11-25T12:04:57,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T12:04:57,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:57,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:57,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:57,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:57,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:57,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536357476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:57,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536357476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:57,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:57,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/20dfc6957fb34927a74ba6fcc86fbc95, entries=150, sequenceid=311, filesize=12.0 K 2024-11-25T12:04:57,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/2fb00e8003944c5c84d2273809091e55 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/2fb00e8003944c5c84d2273809091e55 2024-11-25T12:04:57,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/2fb00e8003944c5c84d2273809091e55, entries=150, sequenceid=311, filesize=12.0 K 2024-11-25T12:04:57,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 5f0aa52f88963b28a40fb54be46c7463 in 788ms, sequenceid=311, compaction requested=true 2024-11-25T12:04:57,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:57,563 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:04:57,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:04:57,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:57,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:04:57,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:57,564 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:04:57,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:04:57,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:57,576 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52108 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:04:57,576 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/A is initiating minor compaction (all files) 2024-11-25T12:04:57,576 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/A in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:57,576 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/a6220b24307f4156aa04ac7628a2a43d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/cde9599b50ed40d7946cc8e6996370bd, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/0190d74acd424bbc9c4a78d3b6da83a6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/a897c2a6c00e4a7f807fae7c0551e0a8] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=50.9 K 2024-11-25T12:04:57,576 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:04:57,576 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/B is initiating minor compaction (all files) 2024-11-25T12:04:57,577 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/B in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:57,577 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/9828ff69c1f445f6b5a446ca1e1a547a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/d24856cf84c34016a276ed56cee9c2c2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/9d70ad28c2b948d5bad1a12884e0cfe6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/20dfc6957fb34927a74ba6fcc86fbc95] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=48.5 K 2024-11-25T12:04:57,577 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting a6220b24307f4156aa04ac7628a2a43d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732536294050 2024-11-25T12:04:57,578 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting cde9599b50ed40d7946cc8e6996370bd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732536294759 2024-11-25T12:04:57,578 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9828ff69c1f445f6b5a446ca1e1a547a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732536294050 2024-11-25T12:04:57,579 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting d24856cf84c34016a276ed56cee9c2c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732536294759 2024-11-25T12:04:57,579 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 0190d74acd424bbc9c4a78d3b6da83a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732536295198 2024-11-25T12:04:57,580 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d70ad28c2b948d5bad1a12884e0cfe6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732536295198 2024-11-25T12:04:57,580 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting a897c2a6c00e4a7f807fae7c0551e0a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732536295600 2024-11-25T12:04:57,580 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20dfc6957fb34927a74ba6fcc86fbc95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732536295600 2024-11-25T12:04:57,597 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#A#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:57,598 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/6e9669a461bb46ba80135cd502469d32 is 50, key is test_row_0/A:col10/1732536295612/Put/seqid=0 2024-11-25T12:04:57,599 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#B#compaction#247 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:57,599 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/ce6b41278a934d059b3dc1e7a55c8e99 is 50, key is test_row_0/B:col10/1732536295612/Put/seqid=0 2024-11-25T12:04:57,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742116_1292 (size=13051) 2024-11-25T12:04:57,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T12:04:57,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742117_1293 (size=13051) 2024-11-25T12:04:57,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-25T12:04:57,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:57,651 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-25T12:04:57,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/c973f1ad58534f7aad383a413478160e is 50, key is test_row_0/A:col10/1732536296825/Put/seqid=0 2024-11-25T12:04:57,670 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/6e9669a461bb46ba80135cd502469d32 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/6e9669a461bb46ba80135cd502469d32 2024-11-25T12:04:57,678 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/A of 5f0aa52f88963b28a40fb54be46c7463 into 6e9669a461bb46ba80135cd502469d32(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:57,678 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:57,678 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/A, priority=12, startTime=1732536297563; duration=0sec 2024-11-25T12:04:57,678 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:57,678 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:A 2024-11-25T12:04:57,678 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:04:57,681 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:04:57,681 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/C is initiating minor compaction (all files) 2024-11-25T12:04:57,681 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/C in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:57,682 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/d0c14977d506454399f5b48d6c66d0ab, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/9a6c01548a9c4aa693c68a1256be6c37, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/cc81c32e773c4725bd4fae25204ce459, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/2fb00e8003944c5c84d2273809091e55] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=48.5 K 2024-11-25T12:04:57,682 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting d0c14977d506454399f5b48d6c66d0ab, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732536294050 2024-11-25T12:04:57,683 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a6c01548a9c4aa693c68a1256be6c37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732536294759 2024-11-25T12:04:57,683 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting cc81c32e773c4725bd4fae25204ce459, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732536295198 2024-11-25T12:04:57,684 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fb00e8003944c5c84d2273809091e55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732536295600 2024-11-25T12:04:57,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742118_1294 (size=12301) 2024-11-25T12:04:57,710 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#C#compaction#249 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:57,711 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/265857f864b24170be8b54d5893006ee is 50, key is test_row_0/C:col10/1732536295612/Put/seqid=0 2024-11-25T12:04:57,715 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/c973f1ad58534f7aad383a413478160e 2024-11-25T12:04:57,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/d930d6c106ac41c7985fb99d27afbcc5 is 50, key is test_row_0/B:col10/1732536296825/Put/seqid=0 2024-11-25T12:04:57,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742119_1295 (size=13051) 2024-11-25T12:04:57,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742120_1296 (size=12301) 2024-11-25T12:04:57,789 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/d930d6c106ac41c7985fb99d27afbcc5 2024-11-25T12:04:57,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/7e0746daaa9648ebb029842593c4eb09 is 50, key is test_row_0/C:col10/1732536296825/Put/seqid=0 2024-11-25T12:04:57,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742121_1297 (size=12301) 2024-11-25T12:04:57,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:57,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:58,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:58,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536358016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:58,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:58,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536358016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:58,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:58,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:58,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:58,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536358021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:58,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:58,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536358023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:58,031 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/ce6b41278a934d059b3dc1e7a55c8e99 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/ce6b41278a934d059b3dc1e7a55c8e99 2024-11-25T12:04:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:58,054 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/B of 5f0aa52f88963b28a40fb54be46c7463 into ce6b41278a934d059b3dc1e7a55c8e99(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:58,054 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:58,054 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/B, priority=12, startTime=1732536297563; duration=0sec 2024-11-25T12:04:58,055 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:58,055 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:B 2024-11-25T12:04:58,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:58,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536358126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:58,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:58,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:58,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536358136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:58,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:58,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:58,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536358136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:58,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:58,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:58,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536358136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:58,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:58,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T12:04:58,156 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/265857f864b24170be8b54d5893006ee as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/265857f864b24170be8b54d5893006ee 2024-11-25T12:04:58,165 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/C of 5f0aa52f88963b28a40fb54be46c7463 into 265857f864b24170be8b54d5893006ee(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:58,165 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:58,165 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/C, priority=12, startTime=1732536297564; duration=0sec 2024-11-25T12:04:58,165 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:58,165 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:C 2024-11-25T12:04:58,253 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/7e0746daaa9648ebb029842593c4eb09 2024-11-25T12:04:58,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/c973f1ad58534f7aad383a413478160e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/c973f1ad58534f7aad383a413478160e 2024-11-25T12:04:58,270 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/c973f1ad58534f7aad383a413478160e, entries=150, sequenceid=330, filesize=12.0 K 2024-11-25T12:04:58,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/d930d6c106ac41c7985fb99d27afbcc5 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/d930d6c106ac41c7985fb99d27afbcc5 2024-11-25T12:04:58,278 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/d930d6c106ac41c7985fb99d27afbcc5, entries=150, sequenceid=330, filesize=12.0 K 2024-11-25T12:04:58,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/7e0746daaa9648ebb029842593c4eb09 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/7e0746daaa9648ebb029842593c4eb09 2024-11-25T12:04:58,287 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/7e0746daaa9648ebb029842593c4eb09, entries=150, sequenceid=330, filesize=12.0 K 2024-11-25T12:04:58,288 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 5f0aa52f88963b28a40fb54be46c7463 in 637ms, sequenceid=330, compaction requested=false 2024-11-25T12:04:58,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:58,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:58,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-25T12:04:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-25T12:04:58,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-25T12:04:58,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2830 sec 2024-11-25T12:04:58,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.3000 sec 2024-11-25T12:04:58,342 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:58,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-25T12:04:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:58,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/9937a7f4136f45ce9a774fa6faef8d88 is 50, key is test_row_0/A:col10/1732536298020/Put/seqid=0 2024-11-25T12:04:58,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742122_1298 (size=12301) 2024-11-25T12:04:58,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:58,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536358388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:58,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:58,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536358398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:58,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536358404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:58,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536358399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:58,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:58,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:58,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536358508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:58,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536358517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:58,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:58,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536358517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:58,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536358519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:58,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:58,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:58,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536358720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:58,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:58,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536358729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:58,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:58,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536358733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:58,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536358736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:58,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:58,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/9937a7f4136f45ce9a774fa6faef8d88 2024-11-25T12:04:58,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/f36a2cb7323545a98c1e4514e79e4533 is 50, key is test_row_0/B:col10/1732536298020/Put/seqid=0 2024-11-25T12:04:58,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742123_1299 (size=12301) 2024-11-25T12:04:58,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/f36a2cb7323545a98c1e4514e79e4533 2024-11-25T12:04:58,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/4714f1d59357405eb7b7457e62263113 is 50, key is test_row_0/C:col10/1732536298020/Put/seqid=0 2024-11-25T12:04:58,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742124_1300 (size=12301) 2024-11-25T12:04:58,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/4714f1d59357405eb7b7457e62263113 2024-11-25T12:04:58,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/9937a7f4136f45ce9a774fa6faef8d88 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/9937a7f4136f45ce9a774fa6faef8d88 2024-11-25T12:04:58,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/9937a7f4136f45ce9a774fa6faef8d88, entries=150, sequenceid=351, filesize=12.0 K 2024-11-25T12:04:58,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/f36a2cb7323545a98c1e4514e79e4533 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/f36a2cb7323545a98c1e4514e79e4533 2024-11-25T12:04:58,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/f36a2cb7323545a98c1e4514e79e4533, entries=150, sequenceid=351, filesize=12.0 K 2024-11-25T12:04:58,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/4714f1d59357405eb7b7457e62263113 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/4714f1d59357405eb7b7457e62263113 2024-11-25T12:04:58,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/4714f1d59357405eb7b7457e62263113, entries=150, sequenceid=351, filesize=12.0 K 2024-11-25T12:04:58,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=114.05 KB/116790 for 5f0aa52f88963b28a40fb54be46c7463 in 552ms, sequenceid=351, compaction requested=true 2024-11-25T12:04:58,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:58,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:04:58,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:58,894 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:58,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:04:58,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:58,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:04:58,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:04:58,895 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:58,896 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:58,896 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:58,897 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/A is initiating minor compaction (all files) 2024-11-25T12:04:58,897 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/B is initiating minor compaction (all files) 2024-11-25T12:04:58,897 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/A in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:58,897 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/B in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:58,897 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/6e9669a461bb46ba80135cd502469d32, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/c973f1ad58534f7aad383a413478160e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/9937a7f4136f45ce9a774fa6faef8d88] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=36.8 K 2024-11-25T12:04:58,897 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/ce6b41278a934d059b3dc1e7a55c8e99, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/d930d6c106ac41c7985fb99d27afbcc5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/f36a2cb7323545a98c1e4514e79e4533] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=36.8 K 2024-11-25T12:04:58,897 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting ce6b41278a934d059b3dc1e7a55c8e99, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732536295600 2024-11-25T12:04:58,897 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e9669a461bb46ba80135cd502469d32, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732536295600 2024-11-25T12:04:58,898 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting d930d6c106ac41c7985fb99d27afbcc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732536296814 2024-11-25T12:04:58,898 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting c973f1ad58534f7aad383a413478160e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732536296814 2024-11-25T12:04:58,898 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting f36a2cb7323545a98c1e4514e79e4533, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732536298014 2024-11-25T12:04:58,898 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9937a7f4136f45ce9a774fa6faef8d88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732536298014 2024-11-25T12:04:58,910 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#B#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:58,910 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/8a0204242e234b8085b4e5db61a260b2 is 50, key is test_row_0/B:col10/1732536298020/Put/seqid=0 2024-11-25T12:04:58,915 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#A#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:58,916 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/99c285c8d92a49299c3f898efc8800f1 is 50, key is test_row_0/A:col10/1732536298020/Put/seqid=0 2024-11-25T12:04:58,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742125_1301 (size=13153) 2024-11-25T12:04:58,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742126_1302 (size=13153) 2024-11-25T12:04:58,966 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/99c285c8d92a49299c3f898efc8800f1 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/99c285c8d92a49299c3f898efc8800f1 2024-11-25T12:04:58,993 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/A of 5f0aa52f88963b28a40fb54be46c7463 into 99c285c8d92a49299c3f898efc8800f1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:58,993 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:58,993 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/A, priority=13, startTime=1732536298894; duration=0sec 2024-11-25T12:04:58,993 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:04:58,993 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:A 2024-11-25T12:04:58,993 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:04:58,995 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:04:58,996 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/C is initiating minor compaction (all files) 2024-11-25T12:04:58,996 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/C in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:58,996 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/265857f864b24170be8b54d5893006ee, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/7e0746daaa9648ebb029842593c4eb09, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/4714f1d59357405eb7b7457e62263113] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=36.8 K 2024-11-25T12:04:58,996 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 265857f864b24170be8b54d5893006ee, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732536295600 2024-11-25T12:04:58,997 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e0746daaa9648ebb029842593c4eb09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732536296814 2024-11-25T12:04:58,997 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4714f1d59357405eb7b7457e62263113, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732536298014 2024-11-25T12:04:59,008 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#C#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:04:59,008 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/8072cfd7066e43e0b50090126a655d62 is 50, key is test_row_0/C:col10/1732536298020/Put/seqid=0 2024-11-25T12:04:59,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742127_1303 (size=13153) 2024-11-25T12:04:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:59,046 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:59,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T12:04:59,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/bb12ededa5b74ad092def08ffc9e0c5b is 50, key is test_row_0/A:col10/1732536298391/Put/seqid=0 2024-11-25T12:04:59,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536359068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536359069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47622 deadline: 1732536359069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536359073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536359077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,089 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/8072cfd7066e43e0b50090126a655d62 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/8072cfd7066e43e0b50090126a655d62 2024-11-25T12:04:59,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742128_1304 (size=17181) 2024-11-25T12:04:59,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/bb12ededa5b74ad092def08ffc9e0c5b 2024-11-25T12:04:59,118 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/C of 5f0aa52f88963b28a40fb54be46c7463 into 8072cfd7066e43e0b50090126a655d62(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:59,118 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:59,118 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/C, priority=13, startTime=1732536298894; duration=0sec 2024-11-25T12:04:59,118 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:59,118 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:C 2024-11-25T12:04:59,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/aaec5ad42d8444fa8bb69977cfaa4ae8 is 50, key is test_row_0/B:col10/1732536298391/Put/seqid=0 2024-11-25T12:04:59,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742129_1305 (size=12301) 2024-11-25T12:04:59,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-25T12:04:59,156 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:04:59,157 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:04:59,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/aaec5ad42d8444fa8bb69977cfaa4ae8 2024-11-25T12:04:59,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-25T12:04:59,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T12:04:59,164 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:04:59,165 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:04:59,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:04:59,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536359176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/0168a5165a2348dc8390f2b4bf50e8ed is 50, key is test_row_0/C:col10/1732536298391/Put/seqid=0 2024-11-25T12:04:59,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536359188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536359188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47622 deadline: 1732536359189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536359189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742130_1306 (size=12301) 2024-11-25T12:04:59,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/0168a5165a2348dc8390f2b4bf50e8ed 2024-11-25T12:04:59,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/bb12ededa5b74ad092def08ffc9e0c5b as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/bb12ededa5b74ad092def08ffc9e0c5b 2024-11-25T12:04:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T12:04:59,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/bb12ededa5b74ad092def08ffc9e0c5b, entries=250, sequenceid=374, filesize=16.8 K 2024-11-25T12:04:59,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/aaec5ad42d8444fa8bb69977cfaa4ae8 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/aaec5ad42d8444fa8bb69977cfaa4ae8 2024-11-25T12:04:59,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/aaec5ad42d8444fa8bb69977cfaa4ae8, entries=150, sequenceid=374, filesize=12.0 K 2024-11-25T12:04:59,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/0168a5165a2348dc8390f2b4bf50e8ed as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/0168a5165a2348dc8390f2b4bf50e8ed 2024-11-25T12:04:59,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T12:04:59,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:59,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:59,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:59,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/0168a5165a2348dc8390f2b4bf50e8ed, entries=150, sequenceid=374, filesize=12.0 K 2024-11-25T12:04:59,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 5f0aa52f88963b28a40fb54be46c7463 in 292ms, sequenceid=374, compaction requested=false 2024-11-25T12:04:59,343 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/8a0204242e234b8085b4e5db61a260b2 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/8a0204242e234b8085b4e5db61a260b2 2024-11-25T12:04:59,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:59,356 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/B of 5f0aa52f88963b28a40fb54be46c7463 into 8a0204242e234b8085b4e5db61a260b2(size=12.8 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:04:59,356 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:04:59,356 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/B, priority=13, startTime=1732536298894; duration=0sec 2024-11-25T12:04:59,357 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:04:59,357 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:B 2024-11-25T12:04:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:04:59,393 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:04:59,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-25T12:04:59,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/ff31f080305a4fd49b292f1e7916fcde is 50, key is test_row_0/A:col10/1732536299391/Put/seqid=0 2024-11-25T12:04:59,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742131_1307 (size=12301) 2024-11-25T12:04:59,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/ff31f080305a4fd49b292f1e7916fcde 2024-11-25T12:04:59,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/7667a67ce13643528cb1043067a88357 is 50, key is test_row_0/B:col10/1732536299391/Put/seqid=0 2024-11-25T12:04:59,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536359461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536359461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47622 deadline: 1732536359463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536359469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536359473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T12:04:59,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T12:04:59,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:59,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:59,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:59,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:59,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:59,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742132_1308 (size=12301) 2024-11-25T12:04:59,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/7667a67ce13643528cb1043067a88357 2024-11-25T12:04:59,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/3d787c43aa804d08b4c5e46b68b63eb6 is 50, key is test_row_0/C:col10/1732536299391/Put/seqid=0 2024-11-25T12:04:59,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742133_1309 (size=12301) 2024-11-25T12:04:59,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536359586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47622 deadline: 1732536359586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536359586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536359587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536359587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T12:04:59,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:59,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:59,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:59,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:59,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:59,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:59,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T12:04:59,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536359797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536359797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47622 deadline: 1732536359797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536359797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:04:59,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536359806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:04:59,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:04:59,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:04:59,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T12:04:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:59,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:59,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:59,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:59,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/3d787c43aa804d08b4c5e46b68b63eb6 2024-11-25T12:04:59,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T12:04:59,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:59,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:04:59,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:04:59,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:59,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:04:59,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/ff31f080305a4fd49b292f1e7916fcde as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/ff31f080305a4fd49b292f1e7916fcde 2024-11-25T12:05:00,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/ff31f080305a4fd49b292f1e7916fcde, entries=150, sequenceid=391, filesize=12.0 K 2024-11-25T12:05:00,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/7667a67ce13643528cb1043067a88357 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/7667a67ce13643528cb1043067a88357 2024-11-25T12:05:00,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/7667a67ce13643528cb1043067a88357, entries=150, sequenceid=391, filesize=12.0 K 2024-11-25T12:05:00,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/3d787c43aa804d08b4c5e46b68b63eb6 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/3d787c43aa804d08b4c5e46b68b63eb6 2024-11-25T12:05:00,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/3d787c43aa804d08b4c5e46b68b63eb6, entries=150, sequenceid=391, filesize=12.0 K 2024-11-25T12:05:00,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 5f0aa52f88963b28a40fb54be46c7463 in 676ms, sequenceid=391, compaction requested=true 2024-11-25T12:05:00,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:05:00,070 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:00,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:05:00,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:00,070 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:00,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:05:00,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:00,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5f0aa52f88963b28a40fb54be46c7463:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:00,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:00,081 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:00,081 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/A is initiating minor compaction (all files) 2024-11-25T12:05:00,081 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/A in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:00,081 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/99c285c8d92a49299c3f898efc8800f1, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/bb12ededa5b74ad092def08ffc9e0c5b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/ff31f080305a4fd49b292f1e7916fcde] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=41.6 K 2024-11-25T12:05:00,082 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:00,082 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/B is initiating minor compaction (all files) 2024-11-25T12:05:00,082 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/B in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:00,082 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/8a0204242e234b8085b4e5db61a260b2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/aaec5ad42d8444fa8bb69977cfaa4ae8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/7667a67ce13643528cb1043067a88357] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=36.9 K 2024-11-25T12:05:00,085 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a0204242e234b8085b4e5db61a260b2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732536298014 2024-11-25T12:05:00,085 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99c285c8d92a49299c3f898efc8800f1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732536298014 2024-11-25T12:05:00,089 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting aaec5ad42d8444fa8bb69977cfaa4ae8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732536298382 2024-11-25T12:05:00,089 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb12ededa5b74ad092def08ffc9e0c5b, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732536298382 2024-11-25T12:05:00,089 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 7667a67ce13643528cb1043067a88357, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732536299068 2024-11-25T12:05:00,091 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff31f080305a4fd49b292f1e7916fcde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732536299068 2024-11-25T12:05:00,122 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#A#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:05:00,123 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:00,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T12:05:00,123 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/8e181f422f20450c853e3be05e477857 is 50, key is test_row_0/A:col10/1732536299391/Put/seqid=0 2024-11-25T12:05:00,128 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#B#compaction#265 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:00,129 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/f8a40e0b63a64be1bc493459694d88c1 is 50, key is test_row_0/B:col10/1732536299391/Put/seqid=0 2024-11-25T12:05:00,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T12:05:00,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:00,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:05:00,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:00,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:00,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:00,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/a93d5c53a8fd43d4a5e5a873de964669 is 50, key is test_row_0/A:col10/1732536299460/Put/seqid=0 2024-11-25T12:05:00,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742134_1310 (size=13255) 2024-11-25T12:05:00,171 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/8e181f422f20450c853e3be05e477857 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/8e181f422f20450c853e3be05e477857 2024-11-25T12:05:00,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742135_1311 (size=12301) 2024-11-25T12:05:00,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742136_1312 (size=13255) 2024-11-25T12:05:00,194 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/A of 5f0aa52f88963b28a40fb54be46c7463 into 8e181f422f20450c853e3be05e477857(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:00,194 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:05:00,194 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/A, priority=13, startTime=1732536300069; duration=0sec 2024-11-25T12:05:00,194 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:00,194 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:A 2024-11-25T12:05:00,194 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:00,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536360180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:00,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:00,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:00,204 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:00,205 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 5f0aa52f88963b28a40fb54be46c7463/C is initiating minor compaction (all files) 2024-11-25T12:05:00,205 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5f0aa52f88963b28a40fb54be46c7463/C in TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:00,205 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/8072cfd7066e43e0b50090126a655d62, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/0168a5165a2348dc8390f2b4bf50e8ed, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/3d787c43aa804d08b4c5e46b68b63eb6] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp, totalSize=36.9 K 2024-11-25T12:05:00,209 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8072cfd7066e43e0b50090126a655d62, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732536298014 2024-11-25T12:05:00,213 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0168a5165a2348dc8390f2b4bf50e8ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732536298382 2024-11-25T12:05:00,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:00,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47622 deadline: 1732536360190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:00,215 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d787c43aa804d08b4c5e46b68b63eb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732536299068 2024-11-25T12:05:00,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536360186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:00,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536360196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:00,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536360213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:00,247 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5f0aa52f88963b28a40fb54be46c7463#C#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:00,256 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/3c807d6d1ab94572a4244e01d9158258 is 50, key is test_row_0/C:col10/1732536299391/Put/seqid=0 2024-11-25T12:05:00,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742137_1313 (size=13255) 2024-11-25T12:05:00,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T12:05:00,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:00,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:05:00,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:00,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:00,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T12:05:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:00,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536360308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:00,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:00,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536360328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:00,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47622 deadline: 1732536360331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:00,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536360341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:00,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536360347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:00,356 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/3c807d6d1ab94572a4244e01d9158258 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/3c807d6d1ab94572a4244e01d9158258 2024-11-25T12:05:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:00,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:00,385 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/C of 5f0aa52f88963b28a40fb54be46c7463 into 3c807d6d1ab94572a4244e01d9158258(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:00,385 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:05:00,385 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/C, priority=13, startTime=1732536300072; duration=0sec 2024-11-25T12:05:00,385 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:00,385 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:C 2024-11-25T12:05:00,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T12:05:00,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:00,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:05:00,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:00,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:00,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:00,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536360526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:00,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:00,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536360546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:00,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:00,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47622 deadline: 1732536360555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:00,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:00,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536360566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:00,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:00,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536360566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:00,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/a93d5c53a8fd43d4a5e5a873de964669 2024-11-25T12:05:00,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/f68097afcbe14d89b5fff49622defe29 is 50, key is test_row_0/B:col10/1732536299460/Put/seqid=0 2024-11-25T12:05:00,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:00,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T12:05:00,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:00,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:05:00,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:00,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:00,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:00,653 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/f8a40e0b63a64be1bc493459694d88c1 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/f8a40e0b63a64be1bc493459694d88c1 2024-11-25T12:05:00,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742138_1314 (size=12301) 2024-11-25T12:05:00,660 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5f0aa52f88963b28a40fb54be46c7463/B of 5f0aa52f88963b28a40fb54be46c7463 into f8a40e0b63a64be1bc493459694d88c1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:00,660 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:05:00,660 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., storeName=5f0aa52f88963b28a40fb54be46c7463/B, priority=13, startTime=1732536300070; duration=0sec 2024-11-25T12:05:00,660 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:00,661 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5f0aa52f88963b28a40fb54be46c7463:B 2024-11-25T12:05:00,774 INFO [Thread-1077 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1077. 2024-11-25T12:05:00,774 DEBUG [Thread-1077 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:00,774 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:00,774 INFO [Thread-1079 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1079. 2024-11-25T12:05:00,774 DEBUG [Thread-1079 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:00,774 DEBUG [Thread-1079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:00,774 DEBUG [Thread-1079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:00,774 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:00,774 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:00,776 INFO [Thread-1075 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1075. 2024-11-25T12:05:00,776 DEBUG [Thread-1075 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:00,776 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:00,776 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:00,776 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:00,778 INFO [Thread-1071 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1071. 2024-11-25T12:05:00,778 DEBUG [Thread-1071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:00,778 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:00,778 DEBUG [Thread-1071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:00,778 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:00,778 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:00,779 INFO [Thread-1073 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1073. 2024-11-25T12:05:00,779 DEBUG [Thread-1073 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:00,779 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:00,779 DEBUG [Thread-1073 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:00,779 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:00,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T12:05:00,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:00,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:05:00,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:00,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:00,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:00,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536360836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:00,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:00,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536360856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:00,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:00,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47622 deadline: 1732536360866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:00,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536360876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:00,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536360896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:00,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T12:05:00,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:00,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:05:00,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:00,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:00,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:01,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/f68097afcbe14d89b5fff49622defe29 2024-11-25T12:05:01,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/781666999e584cec850680c5475c2b58 is 50, key is test_row_0/C:col10/1732536299460/Put/seqid=0 2024-11-25T12:05:01,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742139_1315 (size=12301) 2024-11-25T12:05:01,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T12:05:01,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:01,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:05:01,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:01,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:01,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:01,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T12:05:01,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:01,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:05:01,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:01,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:01,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T12:05:01,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47558 deadline: 1732536361346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:01,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:01,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47622 deadline: 1732536361376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:01,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:01,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47618 deadline: 1732536361377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:01,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:01,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:01,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:01,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47546 deadline: 1732536361386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:01,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47588 deadline: 1732536361407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:01,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T12:05:01,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:01,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:05:01,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:01,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:01,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:01,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/781666999e584cec850680c5475c2b58 2024-11-25T12:05:01,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/a93d5c53a8fd43d4a5e5a873de964669 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/a93d5c53a8fd43d4a5e5a873de964669 2024-11-25T12:05:01,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/a93d5c53a8fd43d4a5e5a873de964669, entries=150, sequenceid=413, filesize=12.0 K 2024-11-25T12:05:01,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T12:05:01,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:01,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. as already flushing 2024-11-25T12:05:01,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:01,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:01,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:01,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/f68097afcbe14d89b5fff49622defe29 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/f68097afcbe14d89b5fff49622defe29 2024-11-25T12:05:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:01,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/f68097afcbe14d89b5fff49622defe29, entries=150, sequenceid=413, filesize=12.0 K 2024-11-25T12:05:01,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/781666999e584cec850680c5475c2b58 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/781666999e584cec850680c5475c2b58 2024-11-25T12:05:01,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/781666999e584cec850680c5475c2b58, entries=150, sequenceid=413, filesize=12.0 K 2024-11-25T12:05:01,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 5f0aa52f88963b28a40fb54be46c7463 in 1481ms, sequenceid=413, compaction requested=false 2024-11-25T12:05:01,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:05:01,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-25T12:05:01,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:01,728 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-25T12:05:01,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/1dbf37220ba1460fb8046003c1f1daf7 is 50, key is test_row_0/A:col10/1732536300149/Put/seqid=0 2024-11-25T12:05:01,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742140_1316 (size=12301) 2024-11-25T12:05:01,788 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/1dbf37220ba1460fb8046003c1f1daf7 2024-11-25T12:05:01,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/144a40661f1b4583b091239d0e310d8f is 50, key is test_row_0/B:col10/1732536300149/Put/seqid=0 2024-11-25T12:05:01,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742141_1317 (size=12301) 2024-11-25T12:05:01,843 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/144a40661f1b4583b091239d0e310d8f 2024-11-25T12:05:01,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/ec91232b5646480fbc9ea068054ad246 is 50, key is test_row_0/C:col10/1732536300149/Put/seqid=0 2024-11-25T12:05:01,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742142_1318 (size=12301) 2024-11-25T12:05:02,284 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/ec91232b5646480fbc9ea068054ad246 2024-11-25T12:05:02,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/1dbf37220ba1460fb8046003c1f1daf7 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/1dbf37220ba1460fb8046003c1f1daf7 2024-11-25T12:05:02,295 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/1dbf37220ba1460fb8046003c1f1daf7, entries=150, sequenceid=431, filesize=12.0 K 2024-11-25T12:05:02,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/144a40661f1b4583b091239d0e310d8f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/144a40661f1b4583b091239d0e310d8f 2024-11-25T12:05:02,301 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/144a40661f1b4583b091239d0e310d8f, entries=150, sequenceid=431, filesize=12.0 K 2024-11-25T12:05:02,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/ec91232b5646480fbc9ea068054ad246 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ec91232b5646480fbc9ea068054ad246 2024-11-25T12:05:02,311 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ec91232b5646480fbc9ea068054ad246, entries=150, sequenceid=431, filesize=12.0 K 2024-11-25T12:05:02,318 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for 5f0aa52f88963b28a40fb54be46c7463 in 590ms, sequenceid=431, compaction requested=true 2024-11-25T12:05:02,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 5f0aa52f88963b28a40fb54be46c7463: 2024-11-25T12:05:02,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:02,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-25T12:05:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-25T12:05:02,324 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-25T12:05:02,324 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1570 sec 2024-11-25T12:05:02,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 3.1690 sec 2024-11-25T12:05:02,359 INFO [Thread-1062 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1062. 2024-11-25T12:05:02,359 DEBUG [Thread-1062 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:02,359 DEBUG [Thread-1062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:02,359 DEBUG [Thread-1062 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:02,359 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:02,393 INFO [Thread-1064 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1064. 2024-11-25T12:05:02,393 DEBUG [Thread-1064 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:02,393 DEBUG [Thread-1064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:02,394 DEBUG [Thread-1064 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:02,396 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:02,396 INFO [Thread-1066 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1066. 2024-11-25T12:05:02,396 DEBUG [Thread-1066 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:02,396 DEBUG [Thread-1066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:02,397 DEBUG [Thread-1066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:02,400 INFO [Thread-1060 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1060. 2024-11-25T12:05:02,400 DEBUG [Thread-1060 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:02,400 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:02,400 DEBUG [Thread-1060 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:02,400 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:02,400 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:02,417 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T12:05:02,419 INFO [Thread-1068 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1068. 2024-11-25T12:05:02,419 DEBUG [Thread-1068 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:02,419 DEBUG [Thread-1068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:02,419 DEBUG [Thread-1068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:02,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-25T12:05:03,357 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:03,360 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T12:05:03,360 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-25T12:05:03,360 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-25T12:05:03,360 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-25T12:05:03,360 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-25T12:05:03,360 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-25T12:05:03,360 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T12:05:03,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1834 2024-11-25T12:05:03,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1781 2024-11-25T12:05:03,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1783 2024-11-25T12:05:03,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1821 2024-11-25T12:05:03,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1770 2024-11-25T12:05:03,361 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T12:05:03,361 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T12:05:03,361 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:05:03,361 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:03,361 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:03,361 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:03,365 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-25T12:05:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:03,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T12:05:03,384 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536303380"}]},"ts":"1732536303380"} 2024-11-25T12:05:03,400 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T12:05:03,400 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T12:05:03,405 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T12:05:03,417 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f0aa52f88963b28a40fb54be46c7463, UNASSIGN}] 2024-11-25T12:05:03,421 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f0aa52f88963b28a40fb54be46c7463, UNASSIGN 2024-11-25T12:05:03,429 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=5f0aa52f88963b28a40fb54be46c7463, regionState=CLOSING, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:03,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f0aa52f88963b28a40fb54be46c7463, UNASSIGN because future has completed 2024-11-25T12:05:03,432 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T12:05:03,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674}] 2024-11-25T12:05:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T12:05:03,585 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:05:03,585 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T12:05:03,585 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing 5f0aa52f88963b28a40fb54be46c7463, disabling compactions & flushes 2024-11-25T12:05:03,585 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:03,585 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:03,585 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. after waiting 0 ms 2024-11-25T12:05:03,585 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:03,585 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(2902): Flushing 5f0aa52f88963b28a40fb54be46c7463 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-25T12:05:03,591 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/4420cee576c84b7a9cf6444a45cc88d9 is 50, key is test_row_0/A:col10/1732536302389/Put/seqid=0 2024-11-25T12:05:03,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742143_1319 (size=12301) 2024-11-25T12:05:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T12:05:04,004 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/4420cee576c84b7a9cf6444a45cc88d9 2024-11-25T12:05:04,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T12:05:04,010 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/c43e681e1dae4cebb7e038a158fc9b29 is 50, key is test_row_0/B:col10/1732536302389/Put/seqid=0 2024-11-25T12:05:04,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742144_1320 (size=12301) 2024-11-25T12:05:04,019 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/c43e681e1dae4cebb7e038a158fc9b29 2024-11-25T12:05:04,026 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/bb6a11ec6bca4ad79cb7bba1e2b2cbdb is 50, key is test_row_0/C:col10/1732536302389/Put/seqid=0 2024-11-25T12:05:04,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742145_1321 (size=12301) 2024-11-25T12:05:04,029 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/bb6a11ec6bca4ad79cb7bba1e2b2cbdb 2024-11-25T12:05:04,033 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/A/4420cee576c84b7a9cf6444a45cc88d9 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/4420cee576c84b7a9cf6444a45cc88d9 2024-11-25T12:05:04,037 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/4420cee576c84b7a9cf6444a45cc88d9, entries=150, sequenceid=439, filesize=12.0 K 2024-11-25T12:05:04,037 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/B/c43e681e1dae4cebb7e038a158fc9b29 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/c43e681e1dae4cebb7e038a158fc9b29 2024-11-25T12:05:04,041 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/c43e681e1dae4cebb7e038a158fc9b29, entries=150, sequenceid=439, filesize=12.0 K 2024-11-25T12:05:04,042 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/.tmp/C/bb6a11ec6bca4ad79cb7bba1e2b2cbdb as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/bb6a11ec6bca4ad79cb7bba1e2b2cbdb 2024-11-25T12:05:04,045 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/bb6a11ec6bca4ad79cb7bba1e2b2cbdb, entries=150, sequenceid=439, filesize=12.0 K 2024-11-25T12:05:04,045 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 5f0aa52f88963b28a40fb54be46c7463 in 460ms, sequenceid=439, compaction requested=true 2024-11-25T12:05:04,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/e7b6217967ef4cedb1edd0fd8c5ac47f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/65e44d459dae4adbae5d980c7b44ca81, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/906c216daa284a959b4ae2f5b3eed5c0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/712e028a104a43a6ab5d04cd6f9a36eb, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/5dc7e23630a3495b8466611bbe7d5152, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/e8eb4892a4b44e8fb5dd688bee818fbf, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/fd67103b0b7c45e6ad4ec954251c0fa3, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/e8bbbc896df642548ec925c06fd98676, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/cdd50de6484344e8b45fc7214ff86328, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/522aabe58b82486683ab26672b355673, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/8af1c9752b8b4be29e4147a0794eacd5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/7bb3767e0d3f48f9a6284fe80491062a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/7ba15caeb23644e688df33c9d287143f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/17177deb3f2b4242a41fb6537eb4d33d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/745fad038e2348f8947db2fb36d546e5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/202703593ece4de9bb3e1b9879de8c67, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/2db0c51700f049f1b34f95556c4cd2d6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/a6220b24307f4156aa04ac7628a2a43d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/98f42c7e00514a6c97628deb6e71514f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/cde9599b50ed40d7946cc8e6996370bd, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/0190d74acd424bbc9c4a78d3b6da83a6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/6e9669a461bb46ba80135cd502469d32, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/a897c2a6c00e4a7f807fae7c0551e0a8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/c973f1ad58534f7aad383a413478160e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/99c285c8d92a49299c3f898efc8800f1, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/9937a7f4136f45ce9a774fa6faef8d88, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/bb12ededa5b74ad092def08ffc9e0c5b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/ff31f080305a4fd49b292f1e7916fcde] to archive 2024-11-25T12:05:04,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:05:04,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/e7b6217967ef4cedb1edd0fd8c5ac47f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/e7b6217967ef4cedb1edd0fd8c5ac47f 2024-11-25T12:05:04,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/65e44d459dae4adbae5d980c7b44ca81 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/65e44d459dae4adbae5d980c7b44ca81 2024-11-25T12:05:04,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/906c216daa284a959b4ae2f5b3eed5c0 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/906c216daa284a959b4ae2f5b3eed5c0 2024-11-25T12:05:04,052 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/712e028a104a43a6ab5d04cd6f9a36eb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/712e028a104a43a6ab5d04cd6f9a36eb 2024-11-25T12:05:04,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/5dc7e23630a3495b8466611bbe7d5152 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/5dc7e23630a3495b8466611bbe7d5152 2024-11-25T12:05:04,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/e8eb4892a4b44e8fb5dd688bee818fbf to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/e8eb4892a4b44e8fb5dd688bee818fbf 2024-11-25T12:05:04,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/fd67103b0b7c45e6ad4ec954251c0fa3 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/fd67103b0b7c45e6ad4ec954251c0fa3 2024-11-25T12:05:04,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/e8bbbc896df642548ec925c06fd98676 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/e8bbbc896df642548ec925c06fd98676 2024-11-25T12:05:04,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/cdd50de6484344e8b45fc7214ff86328 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/cdd50de6484344e8b45fc7214ff86328 2024-11-25T12:05:04,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/522aabe58b82486683ab26672b355673 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/522aabe58b82486683ab26672b355673 2024-11-25T12:05:04,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/8af1c9752b8b4be29e4147a0794eacd5 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/8af1c9752b8b4be29e4147a0794eacd5 2024-11-25T12:05:04,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/7bb3767e0d3f48f9a6284fe80491062a to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/7bb3767e0d3f48f9a6284fe80491062a 2024-11-25T12:05:04,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/7ba15caeb23644e688df33c9d287143f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/7ba15caeb23644e688df33c9d287143f 2024-11-25T12:05:04,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/17177deb3f2b4242a41fb6537eb4d33d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/17177deb3f2b4242a41fb6537eb4d33d 2024-11-25T12:05:04,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/745fad038e2348f8947db2fb36d546e5 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/745fad038e2348f8947db2fb36d546e5 2024-11-25T12:05:04,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/202703593ece4de9bb3e1b9879de8c67 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/202703593ece4de9bb3e1b9879de8c67 2024-11-25T12:05:04,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/2db0c51700f049f1b34f95556c4cd2d6 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/2db0c51700f049f1b34f95556c4cd2d6 2024-11-25T12:05:04,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/a6220b24307f4156aa04ac7628a2a43d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/a6220b24307f4156aa04ac7628a2a43d 2024-11-25T12:05:04,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/98f42c7e00514a6c97628deb6e71514f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/98f42c7e00514a6c97628deb6e71514f 2024-11-25T12:05:04,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/cde9599b50ed40d7946cc8e6996370bd to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/cde9599b50ed40d7946cc8e6996370bd 2024-11-25T12:05:04,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/0190d74acd424bbc9c4a78d3b6da83a6 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/0190d74acd424bbc9c4a78d3b6da83a6 2024-11-25T12:05:04,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/6e9669a461bb46ba80135cd502469d32 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/6e9669a461bb46ba80135cd502469d32 2024-11-25T12:05:04,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/a897c2a6c00e4a7f807fae7c0551e0a8 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/a897c2a6c00e4a7f807fae7c0551e0a8 2024-11-25T12:05:04,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/c973f1ad58534f7aad383a413478160e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/c973f1ad58534f7aad383a413478160e 2024-11-25T12:05:04,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/99c285c8d92a49299c3f898efc8800f1 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/99c285c8d92a49299c3f898efc8800f1 2024-11-25T12:05:04,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/9937a7f4136f45ce9a774fa6faef8d88 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/9937a7f4136f45ce9a774fa6faef8d88 2024-11-25T12:05:04,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/bb12ededa5b74ad092def08ffc9e0c5b to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/bb12ededa5b74ad092def08ffc9e0c5b 2024-11-25T12:05:04,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/ff31f080305a4fd49b292f1e7916fcde to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/ff31f080305a4fd49b292f1e7916fcde 2024-11-25T12:05:04,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/b52b872dd24e4e54a8703af519aba38e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/edf0f3640c3f49c099949c7860169326, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/6885fae639854f0190dfee976e8ec6a8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/bad01e1a938a4c91bf8bce03b8330e69, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/13d45ce6665d4cda93781c97052b6027, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/db27b13b7fbf44208a4978d16fabf852, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/9599c4abb71446e489282a086671f8cf, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/f007806087fc4e83b6ac6e54d66129f6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/e296abe7f03a4530a2ba435700d7c753, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/6ba9a1e3cb8643cb813b7b2ddb7cc8e0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/d96eb73acdf048ac862f64e1251d2612, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/fb12173e3d5c465fb9af9b4e3e11fc21, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/c876f4295a124087847a4b80b870079f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/3121ff1583e045a6a1a6dc38de2f1959, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/b048ec2bc386406c9de526ce52f0ee50, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/21d7f74d77ad40138528827c5a6b9f1f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/3a9cdd2eebdd47298147f39992c143a6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/9828ff69c1f445f6b5a446ca1e1a547a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/22c7c87341934cf7b646c4aa68c65ae3, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/d24856cf84c34016a276ed56cee9c2c2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/9d70ad28c2b948d5bad1a12884e0cfe6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/ce6b41278a934d059b3dc1e7a55c8e99, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/20dfc6957fb34927a74ba6fcc86fbc95, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/d930d6c106ac41c7985fb99d27afbcc5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/8a0204242e234b8085b4e5db61a260b2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/f36a2cb7323545a98c1e4514e79e4533, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/aaec5ad42d8444fa8bb69977cfaa4ae8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/7667a67ce13643528cb1043067a88357] to archive 2024-11-25T12:05:04,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:05:04,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/b52b872dd24e4e54a8703af519aba38e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/b52b872dd24e4e54a8703af519aba38e 2024-11-25T12:05:04,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/edf0f3640c3f49c099949c7860169326 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/edf0f3640c3f49c099949c7860169326 2024-11-25T12:05:04,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/6885fae639854f0190dfee976e8ec6a8 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/6885fae639854f0190dfee976e8ec6a8 2024-11-25T12:05:04,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/bad01e1a938a4c91bf8bce03b8330e69 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/bad01e1a938a4c91bf8bce03b8330e69 2024-11-25T12:05:04,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/13d45ce6665d4cda93781c97052b6027 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/13d45ce6665d4cda93781c97052b6027 2024-11-25T12:05:04,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/db27b13b7fbf44208a4978d16fabf852 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/db27b13b7fbf44208a4978d16fabf852 2024-11-25T12:05:04,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/9599c4abb71446e489282a086671f8cf to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/9599c4abb71446e489282a086671f8cf 2024-11-25T12:05:04,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/f007806087fc4e83b6ac6e54d66129f6 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/f007806087fc4e83b6ac6e54d66129f6 2024-11-25T12:05:04,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/e296abe7f03a4530a2ba435700d7c753 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/e296abe7f03a4530a2ba435700d7c753 2024-11-25T12:05:04,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/6ba9a1e3cb8643cb813b7b2ddb7cc8e0 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/6ba9a1e3cb8643cb813b7b2ddb7cc8e0 2024-11-25T12:05:04,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/d96eb73acdf048ac862f64e1251d2612 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/d96eb73acdf048ac862f64e1251d2612 2024-11-25T12:05:04,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/fb12173e3d5c465fb9af9b4e3e11fc21 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/fb12173e3d5c465fb9af9b4e3e11fc21 2024-11-25T12:05:04,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/c876f4295a124087847a4b80b870079f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/c876f4295a124087847a4b80b870079f 2024-11-25T12:05:04,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/3121ff1583e045a6a1a6dc38de2f1959 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/3121ff1583e045a6a1a6dc38de2f1959 2024-11-25T12:05:04,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/b048ec2bc386406c9de526ce52f0ee50 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/b048ec2bc386406c9de526ce52f0ee50 2024-11-25T12:05:04,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/21d7f74d77ad40138528827c5a6b9f1f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/21d7f74d77ad40138528827c5a6b9f1f 2024-11-25T12:05:04,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/3a9cdd2eebdd47298147f39992c143a6 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/3a9cdd2eebdd47298147f39992c143a6 2024-11-25T12:05:04,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/9828ff69c1f445f6b5a446ca1e1a547a to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/9828ff69c1f445f6b5a446ca1e1a547a 2024-11-25T12:05:04,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/22c7c87341934cf7b646c4aa68c65ae3 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/22c7c87341934cf7b646c4aa68c65ae3 2024-11-25T12:05:04,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/d24856cf84c34016a276ed56cee9c2c2 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/d24856cf84c34016a276ed56cee9c2c2 2024-11-25T12:05:04,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/9d70ad28c2b948d5bad1a12884e0cfe6 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/9d70ad28c2b948d5bad1a12884e0cfe6 2024-11-25T12:05:04,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/ce6b41278a934d059b3dc1e7a55c8e99 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/ce6b41278a934d059b3dc1e7a55c8e99 2024-11-25T12:05:04,135 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/20dfc6957fb34927a74ba6fcc86fbc95 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/20dfc6957fb34927a74ba6fcc86fbc95 2024-11-25T12:05:04,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/d930d6c106ac41c7985fb99d27afbcc5 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/d930d6c106ac41c7985fb99d27afbcc5 2024-11-25T12:05:04,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/8a0204242e234b8085b4e5db61a260b2 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/8a0204242e234b8085b4e5db61a260b2 2024-11-25T12:05:04,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/f36a2cb7323545a98c1e4514e79e4533 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/f36a2cb7323545a98c1e4514e79e4533 2024-11-25T12:05:04,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/aaec5ad42d8444fa8bb69977cfaa4ae8 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/aaec5ad42d8444fa8bb69977cfaa4ae8 2024-11-25T12:05:04,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/7667a67ce13643528cb1043067a88357 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/7667a67ce13643528cb1043067a88357 2024-11-25T12:05:04,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/57d1bedabf8642ac982c6ac441a5d1a8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/3f479537cfb0438a845eba912f0ab372, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/c2b7160087aa471296964e42aba06125, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/10ecff3a013f4497adc25668a2611adb, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/263db9617fe240ea98175161e49ad131, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/e8554aef716943099f54dea67ef0e201, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/8accc138fad14d0ea9daeaabfe797296, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ae69f317e2094cab8572482b1e9d4570, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ddcca5cd9972453e8e6a946588507bb5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/492433cccfbb4a479e891dec45e909b2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ca6a8b7605474f20ac4c98dcd76143be, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/a3960e0c40e64fdc8bea6bd18d77465c, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/f4c2cc027f694d428646d97bbad0bb22, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/adea093cd426441fa69fc2e4c599e698, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/a0a8edf57b6e40b5bfa6c371873f428a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/88067c9030ac4f38ae5925b8e94bc2ea, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/11c5ced748d94a558aedc7174b39261e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/d0c14977d506454399f5b48d6c66d0ab, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ab92773241244c37a6e2f3b9b764fb75, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/9a6c01548a9c4aa693c68a1256be6c37, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/cc81c32e773c4725bd4fae25204ce459, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/265857f864b24170be8b54d5893006ee, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/2fb00e8003944c5c84d2273809091e55, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/7e0746daaa9648ebb029842593c4eb09, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/8072cfd7066e43e0b50090126a655d62, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/4714f1d59357405eb7b7457e62263113, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/0168a5165a2348dc8390f2b4bf50e8ed, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/3d787c43aa804d08b4c5e46b68b63eb6] to archive 2024-11-25T12:05:04,166 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:05:04,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/57d1bedabf8642ac982c6ac441a5d1a8 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/57d1bedabf8642ac982c6ac441a5d1a8 2024-11-25T12:05:04,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/3f479537cfb0438a845eba912f0ab372 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/3f479537cfb0438a845eba912f0ab372 2024-11-25T12:05:04,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/c2b7160087aa471296964e42aba06125 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/c2b7160087aa471296964e42aba06125 2024-11-25T12:05:04,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/10ecff3a013f4497adc25668a2611adb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/10ecff3a013f4497adc25668a2611adb 2024-11-25T12:05:04,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/263db9617fe240ea98175161e49ad131 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/263db9617fe240ea98175161e49ad131 2024-11-25T12:05:04,175 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/e8554aef716943099f54dea67ef0e201 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/e8554aef716943099f54dea67ef0e201 2024-11-25T12:05:04,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/8accc138fad14d0ea9daeaabfe797296 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/8accc138fad14d0ea9daeaabfe797296 2024-11-25T12:05:04,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ae69f317e2094cab8572482b1e9d4570 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ae69f317e2094cab8572482b1e9d4570 2024-11-25T12:05:04,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ddcca5cd9972453e8e6a946588507bb5 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ddcca5cd9972453e8e6a946588507bb5 2024-11-25T12:05:04,180 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/492433cccfbb4a479e891dec45e909b2 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/492433cccfbb4a479e891dec45e909b2 2024-11-25T12:05:04,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ca6a8b7605474f20ac4c98dcd76143be to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ca6a8b7605474f20ac4c98dcd76143be 2024-11-25T12:05:04,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/a3960e0c40e64fdc8bea6bd18d77465c to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/a3960e0c40e64fdc8bea6bd18d77465c 2024-11-25T12:05:04,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/f4c2cc027f694d428646d97bbad0bb22 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/f4c2cc027f694d428646d97bbad0bb22 2024-11-25T12:05:04,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/adea093cd426441fa69fc2e4c599e698 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/adea093cd426441fa69fc2e4c599e698 2024-11-25T12:05:04,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/a0a8edf57b6e40b5bfa6c371873f428a to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/a0a8edf57b6e40b5bfa6c371873f428a 2024-11-25T12:05:04,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/88067c9030ac4f38ae5925b8e94bc2ea to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/88067c9030ac4f38ae5925b8e94bc2ea 2024-11-25T12:05:04,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/11c5ced748d94a558aedc7174b39261e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/11c5ced748d94a558aedc7174b39261e 2024-11-25T12:05:04,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/d0c14977d506454399f5b48d6c66d0ab to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/d0c14977d506454399f5b48d6c66d0ab 2024-11-25T12:05:04,200 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ab92773241244c37a6e2f3b9b764fb75 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ab92773241244c37a6e2f3b9b764fb75 2024-11-25T12:05:04,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/9a6c01548a9c4aa693c68a1256be6c37 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/9a6c01548a9c4aa693c68a1256be6c37 2024-11-25T12:05:04,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/cc81c32e773c4725bd4fae25204ce459 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/cc81c32e773c4725bd4fae25204ce459 2024-11-25T12:05:04,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/265857f864b24170be8b54d5893006ee to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/265857f864b24170be8b54d5893006ee 2024-11-25T12:05:04,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/2fb00e8003944c5c84d2273809091e55 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/2fb00e8003944c5c84d2273809091e55 2024-11-25T12:05:04,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/7e0746daaa9648ebb029842593c4eb09 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/7e0746daaa9648ebb029842593c4eb09 2024-11-25T12:05:04,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/8072cfd7066e43e0b50090126a655d62 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/8072cfd7066e43e0b50090126a655d62 2024-11-25T12:05:04,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/4714f1d59357405eb7b7457e62263113 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/4714f1d59357405eb7b7457e62263113 2024-11-25T12:05:04,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/0168a5165a2348dc8390f2b4bf50e8ed to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/0168a5165a2348dc8390f2b4bf50e8ed 2024-11-25T12:05:04,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/3d787c43aa804d08b4c5e46b68b63eb6 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/3d787c43aa804d08b4c5e46b68b63eb6 2024-11-25T12:05:04,224 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/recovered.edits/442.seqid, newMaxSeqId=442, maxSeqId=1 2024-11-25T12:05:04,225 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. 2024-11-25T12:05:04,225 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for 5f0aa52f88963b28a40fb54be46c7463: Waiting for close lock at 1732536303585Running coprocessor pre-close hooks at 1732536303585Disabling compacts and flushes for region at 1732536303585Disabling writes for close at 1732536303585Obtaining lock to block concurrent updates at 1732536303585Preparing flush snapshotting stores in 5f0aa52f88963b28a40fb54be46c7463 at 1732536303585Finished memstore snapshotting TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732536303586 (+1 ms)Flushing stores of TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463. at 1732536303586Flushing 5f0aa52f88963b28a40fb54be46c7463/A: creating writer at 1732536303586Flushing 5f0aa52f88963b28a40fb54be46c7463/A: appending metadata at 1732536303590 (+4 ms)Flushing 5f0aa52f88963b28a40fb54be46c7463/A: closing flushed file at 1732536303590Flushing 5f0aa52f88963b28a40fb54be46c7463/B: creating writer at 1732536304008 (+418 ms)Flushing 5f0aa52f88963b28a40fb54be46c7463/B: appending metadata at 1732536304010 (+2 ms)Flushing 5f0aa52f88963b28a40fb54be46c7463/B: closing flushed file at 1732536304010Flushing 5f0aa52f88963b28a40fb54be46c7463/C: creating writer at 1732536304022 (+12 ms)Flushing 5f0aa52f88963b28a40fb54be46c7463/C: appending metadata at 1732536304025 (+3 ms)Flushing 5f0aa52f88963b28a40fb54be46c7463/C: closing flushed file at 1732536304025Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45627b8d: reopening flushed file at 1732536304033 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15149395: reopening flushed file at 1732536304037 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cd7a3a9: reopening flushed file at 1732536304041 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 5f0aa52f88963b28a40fb54be46c7463 in 460ms, sequenceid=439, compaction requested=true at 1732536304045 (+4 ms)Writing region close event to WAL at 1732536304219 (+174 ms)Running coprocessor post-close hooks at 1732536304225 (+6 ms)Closed at 1732536304225 2024-11-25T12:05:04,228 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=5f0aa52f88963b28a40fb54be46c7463, regionState=CLOSED 2024-11-25T12:05:04,229 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed 5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:05:04,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 because future has completed 2024-11-25T12:05:04,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-25T12:05:04,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure 5f0aa52f88963b28a40fb54be46c7463, server=2bf237fd86ca,37643,1732536214674 in 804 msec 2024-11-25T12:05:04,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-25T12:05:04,241 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5f0aa52f88963b28a40fb54be46c7463, UNASSIGN in 821 msec 2024-11-25T12:05:04,245 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-25T12:05:04,245 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 836 msec 2024-11-25T12:05:04,246 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536304246"}]},"ts":"1732536304246"} 2024-11-25T12:05:04,249 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T12:05:04,249 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T12:05:04,254 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T12:05:04,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 893 msec 2024-11-25T12:05:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-25T12:05:04,528 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:04,530 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-25T12:05:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:04,533 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:04,534 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=82, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-25T12:05:04,544 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:05:04,552 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/recovered.edits] 2024-11-25T12:05:04,556 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/1dbf37220ba1460fb8046003c1f1daf7 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/1dbf37220ba1460fb8046003c1f1daf7 2024-11-25T12:05:04,561 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/4420cee576c84b7a9cf6444a45cc88d9 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/4420cee576c84b7a9cf6444a45cc88d9 2024-11-25T12:05:04,569 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/8e181f422f20450c853e3be05e477857 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/8e181f422f20450c853e3be05e477857 2024-11-25T12:05:04,570 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/a93d5c53a8fd43d4a5e5a873de964669 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/A/a93d5c53a8fd43d4a5e5a873de964669 2024-11-25T12:05:04,577 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/144a40661f1b4583b091239d0e310d8f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/144a40661f1b4583b091239d0e310d8f 2024-11-25T12:05:04,589 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/c43e681e1dae4cebb7e038a158fc9b29 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/c43e681e1dae4cebb7e038a158fc9b29 2024-11-25T12:05:04,591 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/f68097afcbe14d89b5fff49622defe29 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/f68097afcbe14d89b5fff49622defe29 2024-11-25T12:05:04,594 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/f8a40e0b63a64be1bc493459694d88c1 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/B/f8a40e0b63a64be1bc493459694d88c1 2024-11-25T12:05:04,600 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/3c807d6d1ab94572a4244e01d9158258 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/3c807d6d1ab94572a4244e01d9158258 2024-11-25T12:05:04,602 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/781666999e584cec850680c5475c2b58 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/781666999e584cec850680c5475c2b58 2024-11-25T12:05:04,604 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/bb6a11ec6bca4ad79cb7bba1e2b2cbdb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/bb6a11ec6bca4ad79cb7bba1e2b2cbdb 2024-11-25T12:05:04,605 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ec91232b5646480fbc9ea068054ad246 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/C/ec91232b5646480fbc9ea068054ad246 2024-11-25T12:05:04,610 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/recovered.edits/442.seqid to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463/recovered.edits/442.seqid 2024-11-25T12:05:04,611 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/5f0aa52f88963b28a40fb54be46c7463 2024-11-25T12:05:04,611 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T12:05:04,614 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=82, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:04,624 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T12:05:04,631 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T12:05:04,636 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=82, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:04,637 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T12:05:04,637 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732536304637"}]},"ts":"9223372036854775807"} 2024-11-25T12:05:04,640 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T12:05:04,640 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5f0aa52f88963b28a40fb54be46c7463, NAME => 'TestAcidGuarantees,,1732536278453.5f0aa52f88963b28a40fb54be46c7463.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T12:05:04,640 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T12:05:04,641 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732536304640"}]},"ts":"9223372036854775807"} 2024-11-25T12:05:04,645 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T12:05:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-25T12:05:04,657 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=82, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:04,666 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 127 msec 2024-11-25T12:05:04,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-25T12:05:04,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T12:05:04,860 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:04,872 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 238) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-514927083_22 at /127.0.0.1:34976 [Waiting for operation #262] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2009888188_22 at /127.0.0.1:34828 [Waiting for operation #332] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1459 (was 1338) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7081 (was 4657) - AvailableMemoryMB LEAK? - 2024-11-25T12:05:04,893 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=1459, ProcessCount=11, AvailableMemoryMB=7074 2024-11-25T12:05:04,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T12:05:04,897 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:05:04,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:04,909 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T12:05:04,909 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:04,910 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 83 2024-11-25T12:05:04,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-25T12:05:04,913 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T12:05:04,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742146_1322 (size=959) 2024-11-25T12:05:04,962 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d 2024-11-25T12:05:05,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-25T12:05:05,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742147_1323 (size=53) 2024-11-25T12:05:05,032 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:05:05,032 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 58c2c29070f1735a85b3dd1edce5016e, disabling compactions & flushes 2024-11-25T12:05:05,032 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:05,032 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:05,032 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. after waiting 0 ms 2024-11-25T12:05:05,032 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:05,032 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:05,032 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 58c2c29070f1735a85b3dd1edce5016e: Waiting for close lock at 1732536305032Disabling compacts and flushes for region at 1732536305032Disabling writes for close at 1732536305032Writing region close event to WAL at 1732536305032Closed at 1732536305032 2024-11-25T12:05:05,035 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T12:05:05,036 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732536305035"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732536305035"}]},"ts":"1732536305035"} 2024-11-25T12:05:05,044 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T12:05:05,045 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T12:05:05,046 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536305045"}]},"ts":"1732536305045"} 2024-11-25T12:05:05,049 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T12:05:05,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58c2c29070f1735a85b3dd1edce5016e, ASSIGN}] 2024-11-25T12:05:05,056 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58c2c29070f1735a85b3dd1edce5016e, ASSIGN 2024-11-25T12:05:05,057 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58c2c29070f1735a85b3dd1edce5016e, ASSIGN; state=OFFLINE, location=2bf237fd86ca,37643,1732536214674; forceNewPlan=false, retain=false 2024-11-25T12:05:05,208 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=58c2c29070f1735a85b3dd1edce5016e, regionState=OPENING, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:05,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58c2c29070f1735a85b3dd1edce5016e, ASSIGN because future has completed 2024-11-25T12:05:05,210 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674}] 2024-11-25T12:05:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-25T12:05:05,383 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:05,383 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7752): Opening region: {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:05:05,384 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:05,384 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:05:05,384 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7794): checking encryption for 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:05,384 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7797): checking classloading for 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:05,397 INFO [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:05,402 INFO [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58c2c29070f1735a85b3dd1edce5016e columnFamilyName A 2024-11-25T12:05:05,402 DEBUG [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:05,403 INFO [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] regionserver.HStore(327): Store=58c2c29070f1735a85b3dd1edce5016e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:05:05,403 INFO [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:05,405 INFO [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58c2c29070f1735a85b3dd1edce5016e columnFamilyName B 2024-11-25T12:05:05,405 DEBUG [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:05,406 INFO [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] regionserver.HStore(327): Store=58c2c29070f1735a85b3dd1edce5016e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:05:05,406 INFO [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:05,408 INFO [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58c2c29070f1735a85b3dd1edce5016e columnFamilyName C 2024-11-25T12:05:05,408 DEBUG [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:05,409 INFO [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] regionserver.HStore(327): Store=58c2c29070f1735a85b3dd1edce5016e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:05:05,409 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1038): replaying wal for 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:05,410 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:05,410 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:05,411 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1048): stopping wal replay for 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:05,411 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1060): Cleaning up temporary data for 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:05,412 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:05:05,413 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1093): writing seq id for 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:05,417 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:05:05,418 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1114): Opened 58c2c29070f1735a85b3dd1edce5016e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67668473, jitterRate=0.008338823914527893}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:05:05,418 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:05,419 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1006): Region open journal for 58c2c29070f1735a85b3dd1edce5016e: Running coprocessor pre-open hook at 1732536305384Writing region info on filesystem at 1732536305384Initializing all the Stores at 1732536305388 (+4 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536305389 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536305392 (+3 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536305392Cleaning up temporary data from old regions at 1732536305411 (+19 ms)Running coprocessor post-open hooks at 1732536305418 (+7 ms)Region opened successfully at 1732536305419 (+1 ms) 2024-11-25T12:05:05,421 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., pid=85, masterSystemTime=1732536305372 2024-11-25T12:05:05,433 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:05,433 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:05,434 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=58c2c29070f1735a85b3dd1edce5016e, regionState=OPEN, openSeqNum=2, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:05,437 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 because future has completed 2024-11-25T12:05:05,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-25T12:05:05,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 in 228 msec 2024-11-25T12:05:05,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-25T12:05:05,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58c2c29070f1735a85b3dd1edce5016e, ASSIGN in 392 msec 2024-11-25T12:05:05,446 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T12:05:05,447 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536305446"}]},"ts":"1732536305446"} 2024-11-25T12:05:05,449 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T12:05:05,450 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T12:05:05,452 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 553 msec 2024-11-25T12:05:05,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-25T12:05:05,545 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:05,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4502d66f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:05,548 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:05,548 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:05,555 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bb6fd7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:05,568 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45eca5c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:05,570 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:05,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:05,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:05,573 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:05,574 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T12:05:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 2bf237fd86ca,40019,1732536213609 2024-11-25T12:05:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6f0695ab 2024-11-25T12:05:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T12:05:05,580 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T12:05:05,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T12:05:05,582 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:05:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:05,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742148_1324 (size=995) 2024-11-25T12:05:06,011 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-25T12:05:06,011 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-25T12:05:06,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T12:05:06,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58c2c29070f1735a85b3dd1edce5016e, REOPEN/MOVE}] 2024-11-25T12:05:06,022 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58c2c29070f1735a85b3dd1edce5016e, REOPEN/MOVE 2024-11-25T12:05:06,023 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=58c2c29070f1735a85b3dd1edce5016e, regionState=CLOSING, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:06,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58c2c29070f1735a85b3dd1edce5016e, REOPEN/MOVE because future has completed 2024-11-25T12:05:06,025 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T12:05:06,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674}] 2024-11-25T12:05:06,178 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:06,178 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T12:05:06,178 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 58c2c29070f1735a85b3dd1edce5016e, disabling compactions & flushes 2024-11-25T12:05:06,178 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:06,178 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:06,178 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. after waiting 0 ms 2024-11-25T12:05:06,178 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:06,185 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T12:05:06,193 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:06,193 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 58c2c29070f1735a85b3dd1edce5016e: Waiting for close lock at 1732536306178Running coprocessor pre-close hooks at 1732536306178Disabling compacts and flushes for region at 1732536306178Disabling writes for close at 1732536306178Writing region close event to WAL at 1732536306179 (+1 ms)Running coprocessor post-close hooks at 1732536306193 (+14 ms)Closed at 1732536306193 2024-11-25T12:05:06,193 WARN [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionServer(3299): Not adding moved region record: 58c2c29070f1735a85b3dd1edce5016e to self. 2024-11-25T12:05:06,198 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=58c2c29070f1735a85b3dd1edce5016e, regionState=CLOSED 2024-11-25T12:05:06,201 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:06,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 because future has completed 2024-11-25T12:05:06,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-25T12:05:06,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 in 183 msec 2024-11-25T12:05:06,252 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58c2c29070f1735a85b3dd1edce5016e, REOPEN/MOVE; state=CLOSED, location=2bf237fd86ca,37643,1732536214674; forceNewPlan=false, retain=true 2024-11-25T12:05:06,404 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=58c2c29070f1735a85b3dd1edce5016e, regionState=OPENING, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:06,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58c2c29070f1735a85b3dd1edce5016e, REOPEN/MOVE because future has completed 2024-11-25T12:05:06,411 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674}] 2024-11-25T12:05:06,567 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:06,567 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7752): Opening region: {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:05:06,568 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:06,568 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:05:06,568 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7794): checking encryption for 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:06,568 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7797): checking classloading for 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:06,569 INFO [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:06,576 INFO [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58c2c29070f1735a85b3dd1edce5016e columnFamilyName A 2024-11-25T12:05:06,585 DEBUG [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:06,589 INFO [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] regionserver.HStore(327): Store=58c2c29070f1735a85b3dd1edce5016e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:05:06,590 INFO [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:06,598 INFO [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58c2c29070f1735a85b3dd1edce5016e columnFamilyName B 2024-11-25T12:05:06,598 DEBUG [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:06,604 INFO [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] regionserver.HStore(327): Store=58c2c29070f1735a85b3dd1edce5016e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:05:06,604 INFO [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:06,606 INFO [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58c2c29070f1735a85b3dd1edce5016e columnFamilyName C 2024-11-25T12:05:06,606 DEBUG [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:06,612 INFO [StoreOpener-58c2c29070f1735a85b3dd1edce5016e-1 {}] regionserver.HStore(327): Store=58c2c29070f1735a85b3dd1edce5016e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:05:06,613 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1038): replaying wal for 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:06,617 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:06,622 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:06,624 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1048): stopping wal replay for 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:06,625 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1060): Cleaning up temporary data for 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:06,625 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:05:06,627 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1093): writing seq id for 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:06,629 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1114): Opened 58c2c29070f1735a85b3dd1edce5016e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61668693, jitterRate=-0.0810648649930954}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:05:06,629 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:06,630 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1006): Region open journal for 58c2c29070f1735a85b3dd1edce5016e: Running coprocessor pre-open hook at 1732536306568Writing region info on filesystem at 1732536306568Initializing all the Stores at 1732536306569 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536306569Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536306569Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536306569Cleaning up temporary data from old regions at 1732536306625 (+56 ms)Running coprocessor post-open hooks at 1732536306629 (+4 ms)Region opened successfully at 1732536306630 (+1 ms) 2024-11-25T12:05:06,632 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., pid=90, masterSystemTime=1732536306563 2024-11-25T12:05:06,636 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:06,636 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:06,637 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=58c2c29070f1735a85b3dd1edce5016e, regionState=OPEN, openSeqNum=5, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:06,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 because future has completed 2024-11-25T12:05:06,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=88 2024-11-25T12:05:06,642 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 in 230 msec 2024-11-25T12:05:06,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-25T12:05:06,644 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58c2c29070f1735a85b3dd1edce5016e, REOPEN/MOVE in 621 msec 2024-11-25T12:05:06,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-25T12:05:06,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 634 msec 2024-11-25T12:05:06,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0690 sec 2024-11-25T12:05:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-25T12:05:06,660 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:06,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56947e4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:06,665 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:06,666 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:06,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:06,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e27827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:06,668 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:06,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38dd2f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76cfa173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:06,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:06,670 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:06,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:06,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:06,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@714dafab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:06,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:06,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:06,672 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:06,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7df752c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b38372, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:06,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:06,674 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bfc2a7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:06,676 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:06,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6644d83a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c8520b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,677 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:06,677 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:06,678 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c662a06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:06,680 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:06,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8b8848c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a69c70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:06,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:06,683 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd9be29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:06,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:06,686 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:06,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cd6171, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60619e60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,689 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:06,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:06,689 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:06,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:06,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4492cf68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:06,691 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:06,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ac534a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ffdf7ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,692 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:06,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:06,693 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:06,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:06,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:06,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22f271be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:06,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:06,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:06,695 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44484, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:06,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c15b51e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22721903, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,696 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:06,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:06,698 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd11282, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:06,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:06,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:06,699 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44498, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:06,700 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ae9b386, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71e94d1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,704 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:06,704 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:06,712 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:06,713 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79b33e71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:06,715 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:06,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cc6300f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2efff8d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:06,717 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:06,718 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:06,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:06,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:06,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45025a80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:06,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:06,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:06,719 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bec108, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:06,721 DEBUG [Thread-1486 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:06,721 DEBUG [Thread-1497 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:06,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:06,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:06,722 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:06,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:06,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:05:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:05:06,729 DEBUG [Thread-1488 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:06,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:06,732 DEBUG [Thread-1499 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:06,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:06,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:06,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:06,734 DEBUG [Thread-1492 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:06,734 DEBUG [Thread-1495 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:06,735 DEBUG [Thread-1490 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:06,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:06,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:05:06,737 DEBUG [Thread-1501 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:06,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:06,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:06,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:06,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:06,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:06,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:06,740 DEBUG [Thread-1503 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:06,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:05:06,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:05:06,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:06,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:06,745 DEBUG [Thread-1484 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:06,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:06,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:06,746 DEBUG [Thread-1503 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:06,746 DEBUG [Thread-1503 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:06,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:05:06,753 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:05:06,754 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:06,754 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:06,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:05:06,757 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:06,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-25T12:05:06,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:05:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:05:06,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:05:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:06,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T12:05:06,763 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:06,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T12:05:06,769 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:05:06,770 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:05:06,770 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:05:06,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112578a0e83b8d434b8686122f928ca8fe0f_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_0/A:col10/1732536306755/Put/seqid=0 2024-11-25T12:05:06,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742149_1325 (size=14594) 2024-11-25T12:05:06,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:06,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536366841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:06,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T12:05:06,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536366853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:06,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:06,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:06,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:06,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:06,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536366855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:06,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536366855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:06,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536366857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:06,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:06,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:05:06,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:06,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:06,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:06,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:06,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:06,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:06,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536366967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:06,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536366976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:06,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536366976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:06,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536366976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:06,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:06,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:06,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:06,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:06,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536366987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:06,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:06,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:06,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T12:05:07,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:05:07,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:07,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:07,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:07,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:07,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:07,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:07,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:07,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536367177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:07,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:07,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536367201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:07,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:07,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536367202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:07,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:07,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536367198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:07,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536367213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:07,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:07,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:07,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:05:07,246 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:07,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:07,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:07,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:07,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:07,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:07,283 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112578a0e83b8d434b8686122f928ca8fe0f_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112578a0e83b8d434b8686122f928ca8fe0f_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:07,284 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/963175be125e4c1aac430eeecb40ff76, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:07,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/963175be125e4c1aac430eeecb40ff76 is 175, key is test_row_0/A:col10/1732536306755/Put/seqid=0 2024-11-25T12:05:07,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742150_1326 (size=39549) 2024-11-25T12:05:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T12:05:07,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:05:07,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:07,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:07,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:07,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:07,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:07,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536367497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:07,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536367526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:07,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536367528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:07,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:07,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:07,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:07,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:07,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536367528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:07,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:07,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:07,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:07,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:07,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536367529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:07,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:07,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:07,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:07,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:05:07,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:07,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:07,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:07,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:07,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:07,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:07,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:05:07,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:07,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:07,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:07,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:07,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:07,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:07,721 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/963175be125e4c1aac430eeecb40ff76 2024-11-25T12:05:07,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/8512e73d0d3345e7a6c88c2d010a7426 is 50, key is test_row_0/B:col10/1732536306755/Put/seqid=0 2024-11-25T12:05:07,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:05:07,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:07,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:07,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:07,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:07,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:07,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:07,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742151_1327 (size=12001) 2024-11-25T12:05:07,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T12:05:08,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:08,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536368020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:08,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:05:08,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:08,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:08,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:08,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:08,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:08,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:08,048 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:08,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:08,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:08,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:08,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536368046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:08,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:08,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:08,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:08,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536368046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:08,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:08,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536368049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:08,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:08,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:08,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536368049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:08,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:08,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:08,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:05:08,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:08,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:08,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:08,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:08,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:08,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:08,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/8512e73d0d3345e7a6c88c2d010a7426 2024-11-25T12:05:08,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:05:08,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:08,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:08,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:08,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:08,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:08,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/6c868e90be714b25863e7da9d1ccdb43 is 50, key is test_row_0/C:col10/1732536306755/Put/seqid=0 2024-11-25T12:05:08,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742152_1328 (size=12001) 2024-11-25T12:05:08,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/6c868e90be714b25863e7da9d1ccdb43 2024-11-25T12:05:08,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/963175be125e4c1aac430eeecb40ff76 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/963175be125e4c1aac430eeecb40ff76 2024-11-25T12:05:08,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/963175be125e4c1aac430eeecb40ff76, entries=200, sequenceid=17, filesize=38.6 K 2024-11-25T12:05:08,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/8512e73d0d3345e7a6c88c2d010a7426 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/8512e73d0d3345e7a6c88c2d010a7426 2024-11-25T12:05:08,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/8512e73d0d3345e7a6c88c2d010a7426, entries=150, sequenceid=17, filesize=11.7 K 2024-11-25T12:05:08,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/6c868e90be714b25863e7da9d1ccdb43 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/6c868e90be714b25863e7da9d1ccdb43 2024-11-25T12:05:08,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/6c868e90be714b25863e7da9d1ccdb43, entries=150, sequenceid=17, filesize=11.7 K 2024-11-25T12:05:08,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 58c2c29070f1735a85b3dd1edce5016e in 1697ms, sequenceid=17, compaction requested=false 2024-11-25T12:05:08,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:08,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-25T12:05:08,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:08,522 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T12:05:08,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c368954a1257480b8aaaa0b55d73f1a8_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_0/A:col10/1732536306854/Put/seqid=0 2024-11-25T12:05:08,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742153_1329 (size=12154) 2024-11-25T12:05:08,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,616 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c368954a1257480b8aaaa0b55d73f1a8_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c368954a1257480b8aaaa0b55d73f1a8_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:08,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/0f80a948cae147968af4b9d38c997115, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:08,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/0f80a948cae147968af4b9d38c997115 is 175, key is test_row_0/A:col10/1732536306854/Put/seqid=0 2024-11-25T12:05:08,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742154_1330 (size=30955) 2024-11-25T12:05:08,668 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/0f80a948cae147968af4b9d38c997115 2024-11-25T12:05:08,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/1b2f7ad2bf874076b786bda35e5f3eb9 is 50, key is test_row_0/B:col10/1732536306854/Put/seqid=0 2024-11-25T12:05:08,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742155_1331 (size=12001) 2024-11-25T12:05:08,718 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/1b2f7ad2bf874076b786bda35e5f3eb9 2024-11-25T12:05:08,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/140f6114020a4796ab0f819d9c146b53 is 50, key is test_row_0/C:col10/1732536306854/Put/seqid=0 2024-11-25T12:05:08,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742156_1332 (size=12001) 2024-11-25T12:05:08,804 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/140f6114020a4796ab0f819d9c146b53 2024-11-25T12:05:08,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/0f80a948cae147968af4b9d38c997115 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/0f80a948cae147968af4b9d38c997115 2024-11-25T12:05:08,882 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/0f80a948cae147968af4b9d38c997115, entries=150, sequenceid=41, filesize=30.2 K 2024-11-25T12:05:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/1b2f7ad2bf874076b786bda35e5f3eb9 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/1b2f7ad2bf874076b786bda35e5f3eb9 2024-11-25T12:05:08,890 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/1b2f7ad2bf874076b786bda35e5f3eb9, entries=150, sequenceid=41, filesize=11.7 K 2024-11-25T12:05:08,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/140f6114020a4796ab0f819d9c146b53 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/140f6114020a4796ab0f819d9c146b53 2024-11-25T12:05:08,901 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/140f6114020a4796ab0f819d9c146b53, entries=150, sequenceid=41, filesize=11.7 K 2024-11-25T12:05:08,903 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 58c2c29070f1735a85b3dd1edce5016e in 381ms, sequenceid=41, compaction requested=false 2024-11-25T12:05:08,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:08,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:08,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-25T12:05:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-25T12:05:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T12:05:08,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-25T12:05:08,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1390 sec 2024-11-25T12:05:08,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.1580 sec 2024-11-25T12:05:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:09,416 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:09,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:05:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125068aa1d04df9453e96674fb0be176bc9_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_0/A:col10/1732536309413/Put/seqid=0 2024-11-25T12:05:09,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742158_1334 (size=24358) 2024-11-25T12:05:09,489 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:09,498 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125068aa1d04df9453e96674fb0be176bc9_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125068aa1d04df9453e96674fb0be176bc9_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:09,499 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/580288d0d0064e3ca0f5769d42d051ea, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:09,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/580288d0d0064e3ca0f5769d42d051ea is 175, key is test_row_0/A:col10/1732536309413/Put/seqid=0 2024-11-25T12:05:09,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742157_1333 (size=73995) 2024-11-25T12:05:09,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536369519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:09,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:09,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536369521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:09,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:09,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536369522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:09,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:09,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:09,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536369544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:09,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536369545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:09,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536369657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:09,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536369657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:09,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:09,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536369657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:09,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536369676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:09,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536369676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:09,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:09,815 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T12:05:09,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:09,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536369876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:09,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:09,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536369876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:09,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:09,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536369877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:09,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:09,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536369896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:09,917 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/580288d0d0064e3ca0f5769d42d051ea 2024-11-25T12:05:09,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:09,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:09,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536369896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:09,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:09,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:10,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/ab99aa6a16c3430f836d5c46df28b889 is 50, key is test_row_0/B:col10/1732536309413/Put/seqid=0 2024-11-25T12:05:10,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742159_1335 (size=12001) 2024-11-25T12:05:10,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/ab99aa6a16c3430f836d5c46df28b889 2024-11-25T12:05:10,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/d2ac38eeae8d4a08a93a38ac6eb10934 is 50, key is test_row_0/C:col10/1732536309413/Put/seqid=0 2024-11-25T12:05:10,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742160_1336 (size=12001) 2024-11-25T12:05:10,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/d2ac38eeae8d4a08a93a38ac6eb10934 2024-11-25T12:05:10,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/580288d0d0064e3ca0f5769d42d051ea as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/580288d0d0064e3ca0f5769d42d051ea 2024-11-25T12:05:10,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/580288d0d0064e3ca0f5769d42d051ea, entries=400, sequenceid=52, filesize=72.3 K 2024-11-25T12:05:10,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/ab99aa6a16c3430f836d5c46df28b889 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/ab99aa6a16c3430f836d5c46df28b889 2024-11-25T12:05:10,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/ab99aa6a16c3430f836d5c46df28b889, entries=150, sequenceid=52, filesize=11.7 K 2024-11-25T12:05:10,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/d2ac38eeae8d4a08a93a38ac6eb10934 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/d2ac38eeae8d4a08a93a38ac6eb10934 2024-11-25T12:05:10,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/d2ac38eeae8d4a08a93a38ac6eb10934, entries=150, sequenceid=52, filesize=11.7 K 2024-11-25T12:05:10,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 58c2c29070f1735a85b3dd1edce5016e in 784ms, sequenceid=52, compaction requested=true 2024-11-25T12:05:10,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:10,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:05:10,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:10,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:05:10,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:10,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:10,200 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:10,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:05:10,200 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:10,201 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:10,202 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/B is initiating minor compaction (all files) 2024-11-25T12:05:10,202 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/B in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:10,202 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/8512e73d0d3345e7a6c88c2d010a7426, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/1b2f7ad2bf874076b786bda35e5f3eb9, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/ab99aa6a16c3430f836d5c46df28b889] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=35.2 K 2024-11-25T12:05:10,202 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 144499 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:10,202 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/A is initiating minor compaction (all files) 2024-11-25T12:05:10,202 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/A in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:10,202 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/963175be125e4c1aac430eeecb40ff76, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/0f80a948cae147968af4b9d38c997115, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/580288d0d0064e3ca0f5769d42d051ea] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=141.1 K 2024-11-25T12:05:10,202 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:10,202 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/963175be125e4c1aac430eeecb40ff76, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/0f80a948cae147968af4b9d38c997115, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/580288d0d0064e3ca0f5769d42d051ea] 2024-11-25T12:05:10,203 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 963175be125e4c1aac430eeecb40ff76, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732536306748 2024-11-25T12:05:10,203 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 8512e73d0d3345e7a6c88c2d010a7426, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732536306755 2024-11-25T12:05:10,204 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b2f7ad2bf874076b786bda35e5f3eb9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732536306833 2024-11-25T12:05:10,204 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f80a948cae147968af4b9d38c997115, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732536306833 2024-11-25T12:05:10,204 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting ab99aa6a16c3430f836d5c46df28b889, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732536309237 2024-11-25T12:05:10,204 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 580288d0d0064e3ca0f5769d42d051ea, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732536309229 2024-11-25T12:05:10,245 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:10,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-25T12:05:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:10,263 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:10,263 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/583e04ec00c34cca95a29dc2ea92150c is 50, key is test_row_0/B:col10/1732536309413/Put/seqid=0 2024-11-25T12:05:10,286 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:10,308 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411252446a701a54b4360969a64af714835a6_58c2c29070f1735a85b3dd1edce5016e store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:10,310 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411252446a701a54b4360969a64af714835a6_58c2c29070f1735a85b3dd1edce5016e, store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:10,311 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252446a701a54b4360969a64af714835a6_58c2c29070f1735a85b3dd1edce5016e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:10,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536370253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:10,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:10,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125699f5cf667124bee95e4db2fa1e993c9_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_0/A:col10/1732536310245/Put/seqid=0 2024-11-25T12:05:10,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742161_1337 (size=12104) 2024-11-25T12:05:10,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536370289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:10,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536370305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:10,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:10,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536370310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:10,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536370332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:10,375 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/583e04ec00c34cca95a29dc2ea92150c as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/583e04ec00c34cca95a29dc2ea92150c 2024-11-25T12:05:10,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:10,400 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/B of 58c2c29070f1735a85b3dd1edce5016e into 583e04ec00c34cca95a29dc2ea92150c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:10,401 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:10,401 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/B, priority=13, startTime=1732536310200; duration=0sec 2024-11-25T12:05:10,401 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:10,401 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:B 2024-11-25T12:05:10,401 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:10,402 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:10,402 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/C is initiating minor compaction (all files) 2024-11-25T12:05:10,402 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/C in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:10,402 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/6c868e90be714b25863e7da9d1ccdb43, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/140f6114020a4796ab0f819d9c146b53, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/d2ac38eeae8d4a08a93a38ac6eb10934] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=35.2 K 2024-11-25T12:05:10,407 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c868e90be714b25863e7da9d1ccdb43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732536306755 2024-11-25T12:05:10,408 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 140f6114020a4796ab0f819d9c146b53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732536306833 2024-11-25T12:05:10,408 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting d2ac38eeae8d4a08a93a38ac6eb10934, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732536309237 2024-11-25T12:05:10,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742163_1339 (size=17034) 2024-11-25T12:05:10,429 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,440 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125699f5cf667124bee95e4db2fa1e993c9_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125699f5cf667124bee95e4db2fa1e993c9_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:10,443 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/76469d48e546472a893f28ca783d591b, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:10,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/76469d48e546472a893f28ca783d591b is 175, key is test_row_0/A:col10/1732536310245/Put/seqid=0 2024-11-25T12:05:10,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:10,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742162_1338 (size=4469) 2024-11-25T12:05:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536370420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:10,467 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#A#compaction#286 average throughput is 0.13 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:10,468 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/0d203ab2031444959a295609cbbacd47 is 175, key is test_row_0/A:col10/1732536309413/Put/seqid=0 2024-11-25T12:05:10,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536370462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:10,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,475 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:10,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536370469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:10,489 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#C#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:10,490 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/03d4935112b94db49ddec8adf54d1bf0 is 50, key is test_row_0/C:col10/1732536309413/Put/seqid=0 2024-11-25T12:05:10,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:10,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536370476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:10,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536370486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:10,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:10,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742164_1340 (size=48139) 2024-11-25T12:05:10,560 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/76469d48e546472a893f28ca783d591b 2024-11-25T12:05:10,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742166_1342 (size=12104) 2024-11-25T12:05:10,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742165_1341 (size=31058) 2024-11-25T12:05:10,636 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/0d203ab2031444959a295609cbbacd47 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/0d203ab2031444959a295609cbbacd47 2024-11-25T12:05:10,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/3edd7b51ff2f45b6a4ca09082a50b731 is 50, key is test_row_0/B:col10/1732536310245/Put/seqid=0 2024-11-25T12:05:10,644 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/A of 58c2c29070f1735a85b3dd1edce5016e into 0d203ab2031444959a295609cbbacd47(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:10,644 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:10,645 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/A, priority=13, startTime=1732536310200; duration=0sec 2024-11-25T12:05:10,645 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:10,646 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:A 2024-11-25T12:05:10,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742167_1343 (size=12001) 2024-11-25T12:05:10,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/3edd7b51ff2f45b6a4ca09082a50b731 2024-11-25T12:05:10,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536370676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:10,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536370687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:10,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/c0ab62b235c64fb9a25ccfe5085bf081 is 50, key is test_row_0/C:col10/1732536310245/Put/seqid=0 2024-11-25T12:05:10,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536370696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:10,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:10,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536370706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:10,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536370716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:10,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:10,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742168_1344 (size=12001) 2024-11-25T12:05:10,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/c0ab62b235c64fb9a25ccfe5085bf081 2024-11-25T12:05:10,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/76469d48e546472a893f28ca783d591b as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/76469d48e546472a893f28ca783d591b 2024-11-25T12:05:10,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/76469d48e546472a893f28ca783d591b, entries=250, sequenceid=79, filesize=47.0 K 2024-11-25T12:05:10,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/3edd7b51ff2f45b6a4ca09082a50b731 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/3edd7b51ff2f45b6a4ca09082a50b731 2024-11-25T12:05:10,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/3edd7b51ff2f45b6a4ca09082a50b731, entries=150, sequenceid=79, filesize=11.7 K 2024-11-25T12:05:10,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/c0ab62b235c64fb9a25ccfe5085bf081 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/c0ab62b235c64fb9a25ccfe5085bf081 2024-11-25T12:05:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/c0ab62b235c64fb9a25ccfe5085bf081, entries=150, sequenceid=79, filesize=11.7 K 2024-11-25T12:05:10,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 58c2c29070f1735a85b3dd1edce5016e in 554ms, sequenceid=79, compaction requested=false 2024-11-25T12:05:10,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:10,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-25T12:05:10,929 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:10,933 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:05:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-25T12:05:10,935 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:05:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,936 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:05:10,937 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:05:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T12:05:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,024 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/03d4935112b94db49ddec8adf54d1bf0 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/03d4935112b94db49ddec8adf54d1bf0 2024-11-25T12:05:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,034 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/C of 58c2c29070f1735a85b3dd1edce5016e into 03d4935112b94db49ddec8adf54d1bf0(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:11,035 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:11,035 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/C, priority=13, startTime=1732536310200; duration=0sec 2024-11-25T12:05:11,035 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:11,035 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:C 2024-11-25T12:05:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T12:05:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-25T12:05:11,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:11,096 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-25T12:05:11,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253e31cb31c8604d1b80989f76377e2e97_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_1/A:col10/1732536310305/Put/seqid=0 2024-11-25T12:05:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:11,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:11,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T12:05:11,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742169_1345 (size=9714) 2024-11-25T12:05:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,385 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253e31cb31c8604d1b80989f76377e2e97_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253e31cb31c8604d1b80989f76377e2e97_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:11,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/710147d1ac8e4205a4d33028aedc77e2, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:11,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/710147d1ac8e4205a4d33028aedc77e2 is 175, key is test_row_1/A:col10/1732536310305/Put/seqid=0 2024-11-25T12:05:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742170_1346 (size=22361) 2024-11-25T12:05:11,490 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/710147d1ac8e4205a4d33028aedc77e2 2024-11-25T12:05:11,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/7103b08098c14422ad6954d8ef00ceba is 50, key is test_row_1/B:col10/1732536310305/Put/seqid=0 2024-11-25T12:05:11,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536371510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:11,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:11,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742171_1347 (size=9657) 2024-11-25T12:05:11,572 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/7103b08098c14422ad6954d8ef00ceba 2024-11-25T12:05:11,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536371540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:11,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536371544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:11,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536371548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:11,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536371556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:11,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:11,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T12:05:11,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/85e330c957d34b56a6c33c376a5788f1 is 50, key is test_row_1/C:col10/1732536310305/Put/seqid=0 2024-11-25T12:05:11,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742172_1348 (size=9657) 2024-11-25T12:05:11,658 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/85e330c957d34b56a6c33c376a5788f1 2024-11-25T12:05:11,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/710147d1ac8e4205a4d33028aedc77e2 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/710147d1ac8e4205a4d33028aedc77e2 2024-11-25T12:05:11,690 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/710147d1ac8e4205a4d33028aedc77e2, entries=100, sequenceid=91, filesize=21.8 K 2024-11-25T12:05:11,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/7103b08098c14422ad6954d8ef00ceba as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/7103b08098c14422ad6954d8ef00ceba 2024-11-25T12:05:11,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536371666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:11,706 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/7103b08098c14422ad6954d8ef00ceba, entries=100, sequenceid=91, filesize=9.4 K 2024-11-25T12:05:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:11,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:11,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536371689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:11,712 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:11,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536371689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:11,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/85e330c957d34b56a6c33c376a5788f1 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/85e330c957d34b56a6c33c376a5788f1 2024-11-25T12:05:11,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:11,717 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/85e330c957d34b56a6c33c376a5788f1, entries=100, sequenceid=91, filesize=9.4 K 2024-11-25T12:05:11,721 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 58c2c29070f1735a85b3dd1edce5016e in 625ms, sequenceid=91, compaction requested=true 2024-11-25T12:05:11,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:11,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:11,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-25T12:05:11,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-25T12:05:11,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-25T12:05:11,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 791 msec 2024-11-25T12:05:11,731 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 797 msec 2024-11-25T12:05:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:11,733 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:11,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-25T12:05:11,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c0ecfafa9fad4ad79a0c5f370bbfcc2b_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_0/A:col10/1732536311734/Put/seqid=0 2024-11-25T12:05:11,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536371781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:11,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:11,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536371797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:11,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:11,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742173_1349 (size=12154) 2024-11-25T12:05:11,845 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:11,855 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125c0ecfafa9fad4ad79a0c5f370bbfcc2b_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c0ecfafa9fad4ad79a0c5f370bbfcc2b_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:11,861 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/1db1cbe49caf42d4bf2abe3d4814e52a, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:11,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/1db1cbe49caf42d4bf2abe3d4814e52a is 175, key is test_row_0/A:col10/1732536311734/Put/seqid=0 2024-11-25T12:05:11,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742174_1350 (size=30955) 2024-11-25T12:05:11,913 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/1db1cbe49caf42d4bf2abe3d4814e52a 2024-11-25T12:05:11,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536371906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:11,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:11,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:11,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536371916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:11,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536371916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:11,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:11,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536371917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:11,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:11,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536371917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:11,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/881c5b0ead624a9e9ca0d7a675594658 is 50, key is test_row_0/B:col10/1732536311734/Put/seqid=0 2024-11-25T12:05:11,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742175_1351 (size=12001) 2024-11-25T12:05:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-25T12:05:12,121 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:12,126 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:05:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-25T12:05:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T12:05:12,138 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:05:12,140 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:05:12,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:05:12,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536372136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:12,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:12,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:12,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:12,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536372146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:12,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:12,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:12,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T12:05:12,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:12,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536372236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:12,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:12,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:12,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:12,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536372248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:12,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536372248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:12,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:12,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:12,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:12,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T12:05:12,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:12,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:12,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:12,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:12,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:12,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:12,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/881c5b0ead624a9e9ca0d7a675594658 2024-11-25T12:05:12,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/de528d79cf5547d8a4762d6f7f1a8b24 is 50, key is test_row_0/C:col10/1732536311734/Put/seqid=0 2024-11-25T12:05:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T12:05:12,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T12:05:12,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:12,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:12,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:12,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:12,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:12,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742176_1352 (size=12001) 2024-11-25T12:05:12,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/de528d79cf5547d8a4762d6f7f1a8b24 2024-11-25T12:05:12,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536372464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:12,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:12,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:12,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536372477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:12,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:12,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:12,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:12,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/1db1cbe49caf42d4bf2abe3d4814e52a as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1db1cbe49caf42d4bf2abe3d4814e52a 2024-11-25T12:05:12,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1db1cbe49caf42d4bf2abe3d4814e52a, entries=150, sequenceid=120, filesize=30.2 K 2024-11-25T12:05:12,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/881c5b0ead624a9e9ca0d7a675594658 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/881c5b0ead624a9e9ca0d7a675594658 2024-11-25T12:05:12,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/881c5b0ead624a9e9ca0d7a675594658, entries=150, sequenceid=120, filesize=11.7 K 2024-11-25T12:05:12,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/de528d79cf5547d8a4762d6f7f1a8b24 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/de528d79cf5547d8a4762d6f7f1a8b24 2024-11-25T12:05:12,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/de528d79cf5547d8a4762d6f7f1a8b24, entries=150, sequenceid=120, filesize=11.7 K 2024-11-25T12:05:12,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 58c2c29070f1735a85b3dd1edce5016e in 800ms, sequenceid=120, compaction requested=true 2024-11-25T12:05:12,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:12,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:05:12,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:12,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:05:12,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:05:12,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:12,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:05:12,533 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:12,534 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:12,538 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:12,538 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/C is initiating minor compaction (all files) 2024-11-25T12:05:12,538 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/C in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:12,538 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/03d4935112b94db49ddec8adf54d1bf0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/c0ab62b235c64fb9a25ccfe5085bf081, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/85e330c957d34b56a6c33c376a5788f1, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/de528d79cf5547d8a4762d6f7f1a8b24] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=44.7 K 2024-11-25T12:05:12,544 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132513 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:12,544 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/A is initiating minor compaction (all files) 2024-11-25T12:05:12,545 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/A in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:12,545 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/0d203ab2031444959a295609cbbacd47, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/76469d48e546472a893f28ca783d591b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/710147d1ac8e4205a4d33028aedc77e2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1db1cbe49caf42d4bf2abe3d4814e52a] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=129.4 K 2024-11-25T12:05:12,545 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:12,545 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/0d203ab2031444959a295609cbbacd47, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/76469d48e546472a893f28ca783d591b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/710147d1ac8e4205a4d33028aedc77e2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1db1cbe49caf42d4bf2abe3d4814e52a] 2024-11-25T12:05:12,545 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 03d4935112b94db49ddec8adf54d1bf0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732536309237 2024-11-25T12:05:12,545 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d203ab2031444959a295609cbbacd47, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732536309237 2024-11-25T12:05:12,546 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting c0ab62b235c64fb9a25ccfe5085bf081, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732536309543 2024-11-25T12:05:12,546 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76469d48e546472a893f28ca783d591b, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732536309520 2024-11-25T12:05:12,546 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 85e330c957d34b56a6c33c376a5788f1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732536310301 2024-11-25T12:05:12,546 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 710147d1ac8e4205a4d33028aedc77e2, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732536310301 2024-11-25T12:05:12,547 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting de528d79cf5547d8a4762d6f7f1a8b24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732536311545 2024-11-25T12:05:12,547 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1db1cbe49caf42d4bf2abe3d4814e52a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732536311545 2024-11-25T12:05:12,569 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:12,587 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#C#compaction#298 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:12,588 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/3143cd2f06964a3aa249c8c46f767009 is 50, key is test_row_0/C:col10/1732536311734/Put/seqid=0 2024-11-25T12:05:12,595 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112534648156b6c24a27b3d94496870eba9f_58c2c29070f1735a85b3dd1edce5016e store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:12,598 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112534648156b6c24a27b3d94496870eba9f_58c2c29070f1735a85b3dd1edce5016e, store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:12,598 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112534648156b6c24a27b3d94496870eba9f_58c2c29070f1735a85b3dd1edce5016e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:12,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-25T12:05:12,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:12,625 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-25T12:05:12,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742177_1353 (size=12241) 2024-11-25T12:05:12,654 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/3143cd2f06964a3aa249c8c46f767009 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/3143cd2f06964a3aa249c8c46f767009 2024-11-25T12:05:12,659 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/C of 58c2c29070f1735a85b3dd1edce5016e into 3143cd2f06964a3aa249c8c46f767009(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:12,659 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:12,660 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/C, priority=12, startTime=1732536312533; duration=0sec 2024-11-25T12:05:12,660 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:12,660 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:C 2024-11-25T12:05:12,660 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:12,661 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:12,661 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/B is initiating minor compaction (all files) 2024-11-25T12:05:12,661 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/B in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:12,661 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/583e04ec00c34cca95a29dc2ea92150c, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/3edd7b51ff2f45b6a4ca09082a50b731, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/7103b08098c14422ad6954d8ef00ceba, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/881c5b0ead624a9e9ca0d7a675594658] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=44.7 K 2024-11-25T12:05:12,662 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 583e04ec00c34cca95a29dc2ea92150c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732536309237 2024-11-25T12:05:12,663 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 3edd7b51ff2f45b6a4ca09082a50b731, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732536309543 2024-11-25T12:05:12,665 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 7103b08098c14422ad6954d8ef00ceba, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732536310301 2024-11-25T12:05:12,665 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 881c5b0ead624a9e9ca0d7a675594658, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732536311545 2024-11-25T12:05:12,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742178_1354 (size=4469) 2024-11-25T12:05:12,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125dc275074e8254961b12193e3404c9ebb_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_0/A:col10/1732536311759/Put/seqid=0 2024-11-25T12:05:12,699 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#B#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:12,699 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/f7ed7bef6a2d446eabaa8a912cf00727 is 50, key is test_row_0/B:col10/1732536311734/Put/seqid=0 2024-11-25T12:05:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T12:05:12,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742179_1355 (size=12154) 2024-11-25T12:05:12,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742180_1356 (size=12241) 2024-11-25T12:05:12,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:12,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:12,823 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125dc275074e8254961b12193e3404c9ebb_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125dc275074e8254961b12193e3404c9ebb_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:12,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/c0cd8460afed4b99953029724d68933a, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:12,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/c0cd8460afed4b99953029724d68933a is 175, key is test_row_0/A:col10/1732536311759/Put/seqid=0 2024-11-25T12:05:12,845 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/f7ed7bef6a2d446eabaa8a912cf00727 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/f7ed7bef6a2d446eabaa8a912cf00727 2024-11-25T12:05:12,891 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/B of 58c2c29070f1735a85b3dd1edce5016e into f7ed7bef6a2d446eabaa8a912cf00727(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:12,891 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:12,891 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/B, priority=12, startTime=1732536312533; duration=0sec 2024-11-25T12:05:12,891 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:12,891 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:B 2024-11-25T12:05:12,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742181_1357 (size=30955) 2024-11-25T12:05:13,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536373016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:13,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:13,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536373021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:13,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536373024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:13,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536373026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:13,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:13,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536373052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:13,070 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#A#compaction#297 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:13,070 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/5e30dced5cab47eab4cd5769d1b56b85 is 175, key is test_row_0/A:col10/1732536311734/Put/seqid=0 2024-11-25T12:05:13,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742182_1358 (size=31195) 2024-11-25T12:05:13,174 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/5e30dced5cab47eab4cd5769d1b56b85 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/5e30dced5cab47eab4cd5769d1b56b85 2024-11-25T12:05:13,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:13,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536373157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:13,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:13,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536373161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:13,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536373161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:13,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:13,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536373177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:13,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:13,218 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/A of 58c2c29070f1735a85b3dd1edce5016e into 5e30dced5cab47eab4cd5769d1b56b85(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:13,218 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:13,218 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/A, priority=12, startTime=1732536312533; duration=0sec 2024-11-25T12:05:13,218 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:13,218 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:A 2024-11-25T12:05:13,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T12:05:13,312 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=127, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/c0cd8460afed4b99953029724d68933a 2024-11-25T12:05:13,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/c6d7ced7008945b88fa8154cbb031398 is 50, key is test_row_0/B:col10/1732536311759/Put/seqid=0 2024-11-25T12:05:13,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536373386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:13,414 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:13,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536373387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:13,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536373387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:13,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,422 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:13,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742183_1359 (size=12001) 2024-11-25T12:05:13,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536373416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:13,454 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/c6d7ced7008945b88fa8154cbb031398 2024-11-25T12:05:13,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/16a6d5b858014b9e8a813d6d31dd6434 is 50, key is test_row_0/C:col10/1732536311759/Put/seqid=0 2024-11-25T12:05:13,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742184_1360 (size=12001) 2024-11-25T12:05:13,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536373729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:13,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536373729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:13,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:13,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536373734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:13,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:13,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536373749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:13,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:14,019 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/16a6d5b858014b9e8a813d6d31dd6434 2024-11-25T12:05:14,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/c0cd8460afed4b99953029724d68933a as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/c0cd8460afed4b99953029724d68933a 2024-11-25T12:05:14,032 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/c0cd8460afed4b99953029724d68933a, entries=150, sequenceid=127, filesize=30.2 K 2024-11-25T12:05:14,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/c6d7ced7008945b88fa8154cbb031398 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/c6d7ced7008945b88fa8154cbb031398 2024-11-25T12:05:14,061 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/c6d7ced7008945b88fa8154cbb031398, entries=150, sequenceid=127, filesize=11.7 K 2024-11-25T12:05:14,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/16a6d5b858014b9e8a813d6d31dd6434 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/16a6d5b858014b9e8a813d6d31dd6434 2024-11-25T12:05:14,093 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/16a6d5b858014b9e8a813d6d31dd6434, entries=150, sequenceid=127, filesize=11.7 K 2024-11-25T12:05:14,095 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for 58c2c29070f1735a85b3dd1edce5016e in 1469ms, sequenceid=127, compaction requested=false 2024-11-25T12:05:14,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:14,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:14,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-25T12:05:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-25T12:05:14,116 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-25T12:05:14,117 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9640 sec 2024-11-25T12:05:14,121 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:14,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-25T12:05:14,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:14,128 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.9940 sec 2024-11-25T12:05:14,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ca42635f874149498fcafd29b40dc028_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_0/A:col10/1732536313021/Put/seqid=0 2024-11-25T12:05:14,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742185_1361 (size=14794) 2024-11-25T12:05:14,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536374174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:14,181 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:14,192 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ca42635f874149498fcafd29b40dc028_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ca42635f874149498fcafd29b40dc028_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:14,193 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/3ef16a6a178942d5b9b62f78e116022a, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:14,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/3ef16a6a178942d5b9b62f78e116022a is 175, key is test_row_0/A:col10/1732536313021/Put/seqid=0 2024-11-25T12:05:14,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742186_1362 (size=39749) 2024-11-25T12:05:14,242 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/3ef16a6a178942d5b9b62f78e116022a 2024-11-25T12:05:14,254 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T12:05:14,254 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-25T12:05:14,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/436cf7930a7a4c7bbc19d84487427fb7 is 50, key is test_row_0/B:col10/1732536313021/Put/seqid=0 2024-11-25T12:05:14,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:14,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536374250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:14,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536374250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:14,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:14,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:14,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:14,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:14,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:14,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:14,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:14,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536374256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:14,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:14,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:14,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:14,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-25T12:05:14,286 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:14,287 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:05:14,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-25T12:05:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T12:05:14,289 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:05:14,290 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:05:14,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:05:14,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536374276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:14,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:14,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:14,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:14,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536374286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:14,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:14,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:14,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:14,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742187_1363 (size=12151) 2024-11-25T12:05:14,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/436cf7930a7a4c7bbc19d84487427fb7 2024-11-25T12:05:14,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/164566bd72e44897a7df19bbc97c9121 is 50, key is test_row_0/C:col10/1732536313021/Put/seqid=0 2024-11-25T12:05:14,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742188_1364 (size=12151) 2024-11-25T12:05:14,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/164566bd72e44897a7df19bbc97c9121 2024-11-25T12:05:14,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/3ef16a6a178942d5b9b62f78e116022a as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/3ef16a6a178942d5b9b62f78e116022a 2024-11-25T12:05:14,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/3ef16a6a178942d5b9b62f78e116022a, entries=200, sequenceid=160, filesize=38.8 K 2024-11-25T12:05:14,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/436cf7930a7a4c7bbc19d84487427fb7 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/436cf7930a7a4c7bbc19d84487427fb7 2024-11-25T12:05:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T12:05:14,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/436cf7930a7a4c7bbc19d84487427fb7, entries=150, sequenceid=160, filesize=11.9 K 2024-11-25T12:05:14,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/164566bd72e44897a7df19bbc97c9121 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/164566bd72e44897a7df19bbc97c9121 2024-11-25T12:05:14,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/164566bd72e44897a7df19bbc97c9121, entries=150, sequenceid=160, filesize=11.9 K 2024-11-25T12:05:14,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 58c2c29070f1735a85b3dd1edce5016e in 282ms, sequenceid=160, compaction requested=true 2024-11-25T12:05:14,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:14,404 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:14,405 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:14,405 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/A is initiating minor compaction (all files) 2024-11-25T12:05:14,405 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/A in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:14,405 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/5e30dced5cab47eab4cd5769d1b56b85, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/c0cd8460afed4b99953029724d68933a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/3ef16a6a178942d5b9b62f78e116022a] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=99.5 K 2024-11-25T12:05:14,405 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:14,405 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/5e30dced5cab47eab4cd5769d1b56b85, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/c0cd8460afed4b99953029724d68933a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/3ef16a6a178942d5b9b62f78e116022a] 2024-11-25T12:05:14,405 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e30dced5cab47eab4cd5769d1b56b85, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732536311545 2024-11-25T12:05:14,406 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0cd8460afed4b99953029724d68933a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732536311754 2024-11-25T12:05:14,406 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ef16a6a178942d5b9b62f78e116022a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732536313017 2024-11-25T12:05:14,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:05:14,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:14,419 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:14,420 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:14,420 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/B is initiating minor compaction (all files) 2024-11-25T12:05:14,420 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/B in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:14,420 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/f7ed7bef6a2d446eabaa8a912cf00727, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/c6d7ced7008945b88fa8154cbb031398, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/436cf7930a7a4c7bbc19d84487427fb7] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=35.5 K 2024-11-25T12:05:14,420 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting f7ed7bef6a2d446eabaa8a912cf00727, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732536311545 2024-11-25T12:05:14,421 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:14,427 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting c6d7ced7008945b88fa8154cbb031398, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732536311754 2024-11-25T12:05:14,428 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 436cf7930a7a4c7bbc19d84487427fb7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732536313021 2024-11-25T12:05:14,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:05:14,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:14,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:14,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:14,437 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125b742cb6f249642b78f2b39fe5bcf3922_58c2c29070f1735a85b3dd1edce5016e store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:14,439 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125b742cb6f249642b78f2b39fe5bcf3922_58c2c29070f1735a85b3dd1edce5016e, store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:14,439 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b742cb6f249642b78f2b39fe5bcf3922_58c2c29070f1735a85b3dd1edce5016e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:14,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-25T12:05:14,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:14,461 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-25T12:05:14,465 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#B#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:14,465 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/3b806a1c72ad4e6d83e92c641ae4272e is 50, key is test_row_0/B:col10/1732536313021/Put/seqid=0 2024-11-25T12:05:14,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112510586990cc764c21830469034f108e7e_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_0/A:col10/1732536314172/Put/seqid=0 2024-11-25T12:05:14,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742189_1365 (size=4469) 2024-11-25T12:05:14,506 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#A#compaction#306 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:14,507 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/45c9ffb5ff5b482fa047eefd89af330b is 175, key is test_row_0/A:col10/1732536313021/Put/seqid=0 2024-11-25T12:05:14,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742190_1366 (size=12493) 2024-11-25T12:05:14,581 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/3b806a1c72ad4e6d83e92c641ae4272e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/3b806a1c72ad4e6d83e92c641ae4272e 2024-11-25T12:05:14,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742191_1367 (size=12304) 2024-11-25T12:05:14,587 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/B of 58c2c29070f1735a85b3dd1edce5016e into 3b806a1c72ad4e6d83e92c641ae4272e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:14,587 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:14,587 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/B, priority=13, startTime=1732536314419; duration=0sec 2024-11-25T12:05:14,587 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:14,587 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:B 2024-11-25T12:05:14,587 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:14,588 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:14,588 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/C is initiating minor compaction (all files) 2024-11-25T12:05:14,588 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/C in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:14,589 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/3143cd2f06964a3aa249c8c46f767009, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/16a6d5b858014b9e8a813d6d31dd6434, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/164566bd72e44897a7df19bbc97c9121] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=35.5 K 2024-11-25T12:05:14,589 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 3143cd2f06964a3aa249c8c46f767009, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732536311545 2024-11-25T12:05:14,589 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 16a6d5b858014b9e8a813d6d31dd6434, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732536311754 2024-11-25T12:05:14,591 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 164566bd72e44897a7df19bbc97c9121, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732536313021 2024-11-25T12:05:14,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:14,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742192_1368 (size=31447) 2024-11-25T12:05:14,605 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#C#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:14,606 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/36b5c5f2389d4cd0a6ccc908424c5d8e is 50, key is test_row_0/C:col10/1732536313021/Put/seqid=0 2024-11-25T12:05:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T12:05:14,610 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112510586990cc764c21830469034f108e7e_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112510586990cc764c21830469034f108e7e_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:14,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/1aa5ade524704c048fc41229e129d141, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:14,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/1aa5ade524704c048fc41229e129d141 is 175, key is test_row_0/A:col10/1732536314172/Put/seqid=0 2024-11-25T12:05:14,621 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/45c9ffb5ff5b482fa047eefd89af330b as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/45c9ffb5ff5b482fa047eefd89af330b 2024-11-25T12:05:14,626 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/A of 58c2c29070f1735a85b3dd1edce5016e into 45c9ffb5ff5b482fa047eefd89af330b(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:14,626 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:14,626 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/A, priority=13, startTime=1732536314404; duration=0sec 2024-11-25T12:05:14,627 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:14,627 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:A 2024-11-25T12:05:14,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:14,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742193_1369 (size=31105) 2024-11-25T12:05:14,710 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/1aa5ade524704c048fc41229e129d141 2024-11-25T12:05:14,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/2293763114b349e493bbc87476674191 is 50, key is test_row_0/B:col10/1732536314172/Put/seqid=0 2024-11-25T12:05:14,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742194_1370 (size=12493) 2024-11-25T12:05:14,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742195_1371 (size=12151) 2024-11-25T12:05:14,774 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/2293763114b349e493bbc87476674191 2024-11-25T12:05:14,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/1c22c029fb7e4f0e835b4a61a8ddb2ba is 50, key is test_row_0/C:col10/1732536314172/Put/seqid=0 2024-11-25T12:05:14,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742196_1372 (size=12151) 2024-11-25T12:05:14,858 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/1c22c029fb7e4f0e835b4a61a8ddb2ba 2024-11-25T12:05:14,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/1aa5ade524704c048fc41229e129d141 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1aa5ade524704c048fc41229e129d141 2024-11-25T12:05:14,872 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1aa5ade524704c048fc41229e129d141, entries=150, sequenceid=166, filesize=30.4 K 2024-11-25T12:05:14,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/2293763114b349e493bbc87476674191 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/2293763114b349e493bbc87476674191 2024-11-25T12:05:14,886 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/2293763114b349e493bbc87476674191, entries=150, sequenceid=166, filesize=11.9 K 2024-11-25T12:05:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/1c22c029fb7e4f0e835b4a61a8ddb2ba as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/1c22c029fb7e4f0e835b4a61a8ddb2ba 2024-11-25T12:05:14,892 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/1c22c029fb7e4f0e835b4a61a8ddb2ba, entries=150, sequenceid=166, filesize=11.9 K 2024-11-25T12:05:14,893 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=93.93 KB/96180 for 58c2c29070f1735a85b3dd1edce5016e in 432ms, sequenceid=166, compaction requested=false 2024-11-25T12:05:14,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:14,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:14,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-25T12:05:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-25T12:05:14,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-25T12:05:14,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 610 msec 2024-11-25T12:05:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:14,904 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:14,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-25T12:05:14,907 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 616 msec 2024-11-25T12:05:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-25T12:05:14,917 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:14,917 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:05:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-25T12:05:14,919 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:05:14,921 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:05:14,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:05:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T12:05:14,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411257ea6f91b84904ef29304a3e9371ed1c1_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_0/A:col10/1732536314897/Put/seqid=0 2024-11-25T12:05:14,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742197_1373 (size=14794) 2024-11-25T12:05:14,968 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:14,973 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411257ea6f91b84904ef29304a3e9371ed1c1_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257ea6f91b84904ef29304a3e9371ed1c1_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:14,974 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/2421e91783f346be9a89df78980f4a2c, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:14,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/2421e91783f346be9a89df78980f4a2c is 175, key is test_row_0/A:col10/1732536314897/Put/seqid=0 2024-11-25T12:05:14,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742198_1374 (size=39749) 2024-11-25T12:05:15,001 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/2421e91783f346be9a89df78980f4a2c 2024-11-25T12:05:15,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/ce1d5aec2aee45a2bbbd7ba633756ab1 is 50, key is test_row_0/B:col10/1732536314897/Put/seqid=0 2024-11-25T12:05:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T12:05:15,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742199_1375 (size=12151) 2024-11-25T12:05:15,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T12:05:15,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:15,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:15,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:15,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:15,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:15,164 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/36b5c5f2389d4cd0a6ccc908424c5d8e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/36b5c5f2389d4cd0a6ccc908424c5d8e 2024-11-25T12:05:15,169 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/C of 58c2c29070f1735a85b3dd1edce5016e into 36b5c5f2389d4cd0a6ccc908424c5d8e(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:15,169 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:15,169 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/C, priority=13, startTime=1732536314436; duration=0sec 2024-11-25T12:05:15,170 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:15,170 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:C 2024-11-25T12:05:15,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T12:05:15,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:15,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:15,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:15,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:15,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T12:05:15,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536375276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:15,294 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_2' in 58c2c29070f1735a85b3dd1edce5016e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2268 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:05:15,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:15,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:15,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:15,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:15,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536375282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:15,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:15,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536375287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:15,303 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_0' in 58c2c29070f1735a85b3dd1edce5016e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2282 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:05:15,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:15,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:15,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:15,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536375287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:15,305 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_0' in 58c2c29070f1735a85b3dd1edce5016e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2284 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:05:15,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:15,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:15,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:15,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536375309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:15,340 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_1' in 58c2c29070f1735a85b3dd1edce5016e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2291 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:05:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:15,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T12:05:15,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:15,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:15,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:15,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:15,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:15,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536375406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:15,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/ce1d5aec2aee45a2bbbd7ba633756ab1 2024-11-25T12:05:15,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/2fb35d656c33423eb26bb4e13211df0a is 50, key is test_row_0/C:col10/1732536314897/Put/seqid=0 2024-11-25T12:05:15,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742200_1376 (size=12151) 2024-11-25T12:05:15,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T12:05:15,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:15,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:15,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:15,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:15,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T12:05:15,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/2fb35d656c33423eb26bb4e13211df0a 2024-11-25T12:05:15,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/2421e91783f346be9a89df78980f4a2c as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/2421e91783f346be9a89df78980f4a2c 2024-11-25T12:05:15,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/2421e91783f346be9a89df78980f4a2c, entries=200, sequenceid=186, filesize=38.8 K 2024-11-25T12:05:15,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/ce1d5aec2aee45a2bbbd7ba633756ab1 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/ce1d5aec2aee45a2bbbd7ba633756ab1 2024-11-25T12:05:15,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/ce1d5aec2aee45a2bbbd7ba633756ab1, entries=150, sequenceid=186, filesize=11.9 K 2024-11-25T12:05:15,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/2fb35d656c33423eb26bb4e13211df0a as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/2fb35d656c33423eb26bb4e13211df0a 2024-11-25T12:05:15,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/2fb35d656c33423eb26bb4e13211df0a, entries=150, sequenceid=186, filesize=11.9 K 2024-11-25T12:05:15,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 58c2c29070f1735a85b3dd1edce5016e in 731ms, sequenceid=186, compaction requested=true 2024-11-25T12:05:15,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:15,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:05:15,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:15,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:05:15,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:05:15,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:15,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:05:15,636 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:15,636 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:15,638 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:15,638 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/A is initiating minor compaction (all files) 2024-11-25T12:05:15,638 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/A in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:15,638 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/45c9ffb5ff5b482fa047eefd89af330b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1aa5ade524704c048fc41229e129d141, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/2421e91783f346be9a89df78980f4a2c] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=99.9 K 2024-11-25T12:05:15,639 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:15,639 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/45c9ffb5ff5b482fa047eefd89af330b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1aa5ade524704c048fc41229e129d141, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/2421e91783f346be9a89df78980f4a2c] 2024-11-25T12:05:15,639 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:15,639 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/C is initiating minor compaction (all files) 2024-11-25T12:05:15,639 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/C in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:15,639 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/36b5c5f2389d4cd0a6ccc908424c5d8e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/1c22c029fb7e4f0e835b4a61a8ddb2ba, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/2fb35d656c33423eb26bb4e13211df0a] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=35.9 K 2024-11-25T12:05:15,639 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45c9ffb5ff5b482fa047eefd89af330b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732536313021 2024-11-25T12:05:15,640 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 36b5c5f2389d4cd0a6ccc908424c5d8e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732536313021 2024-11-25T12:05:15,640 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1aa5ade524704c048fc41229e129d141, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732536314141 2024-11-25T12:05:15,644 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c22c029fb7e4f0e835b4a61a8ddb2ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732536314141 2024-11-25T12:05:15,644 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2421e91783f346be9a89df78980f4a2c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732536314763 2024-11-25T12:05:15,645 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fb35d656c33423eb26bb4e13211df0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732536314828 2024-11-25T12:05:15,655 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:15,667 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#C#compaction#316 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:15,667 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/85a77095c82d4bc9bb94f31429fe016b is 50, key is test_row_0/C:col10/1732536314897/Put/seqid=0 2024-11-25T12:05:15,677 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125a14fe2475df541b28e8ab3a5141d9c22_58c2c29070f1735a85b3dd1edce5016e store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:15,680 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125a14fe2475df541b28e8ab3a5141d9c22_58c2c29070f1735a85b3dd1edce5016e, store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:15,680 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a14fe2475df541b28e8ab3a5141d9c22_58c2c29070f1735a85b3dd1edce5016e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:15,695 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:15,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-25T12:05:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:15,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T12:05:15,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:15,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:15,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:15,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:15,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:15,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:15,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742201_1377 (size=12595) 2024-11-25T12:05:15,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112510b7a7d1251d402f8a8cf70a2b635e41_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_0/A:col10/1732536315112/Put/seqid=0 2024-11-25T12:05:15,786 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/85a77095c82d4bc9bb94f31429fe016b as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/85a77095c82d4bc9bb94f31429fe016b 2024-11-25T12:05:15,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742202_1378 (size=4469) 2024-11-25T12:05:15,813 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#A#compaction#315 average throughput is 0.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:15,813 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/C of 58c2c29070f1735a85b3dd1edce5016e into 85a77095c82d4bc9bb94f31429fe016b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:15,813 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:15,813 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/2b0f1ff29b544035a7705e8bf8428d58 is 175, key is test_row_0/A:col10/1732536314897/Put/seqid=0 2024-11-25T12:05:15,814 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/C, priority=13, startTime=1732536315636; duration=0sec 2024-11-25T12:05:15,814 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:15,814 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:C 2024-11-25T12:05:15,814 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:15,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742203_1379 (size=14794) 2024-11-25T12:05:15,833 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:15,833 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/B is initiating minor compaction (all files) 2024-11-25T12:05:15,833 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/B in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:15,833 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/3b806a1c72ad4e6d83e92c641ae4272e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/2293763114b349e493bbc87476674191, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/ce1d5aec2aee45a2bbbd7ba633756ab1] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=35.9 K 2024-11-25T12:05:15,837 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b806a1c72ad4e6d83e92c641ae4272e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732536313021 2024-11-25T12:05:15,848 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 2293763114b349e493bbc87476674191, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732536314141 2024-11-25T12:05:15,850 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting ce1d5aec2aee45a2bbbd7ba633756ab1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732536314828 2024-11-25T12:05:15,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742204_1380 (size=31549) 2024-11-25T12:05:15,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T12:05:15,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:15,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:15,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:15,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:15,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:15,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:15,899 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/2b0f1ff29b544035a7705e8bf8428d58 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/2b0f1ff29b544035a7705e8bf8428d58 2024-11-25T12:05:15,905 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#B#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:15,905 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/3277de8df2aa455b95902b41087cfb97 is 50, key is test_row_0/B:col10/1732536314897/Put/seqid=0 2024-11-25T12:05:15,928 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/A of 58c2c29070f1735a85b3dd1edce5016e into 2b0f1ff29b544035a7705e8bf8428d58(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:15,928 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:15,928 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/A, priority=13, startTime=1732536315636; duration=0sec 2024-11-25T12:05:15,929 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:15,929 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:A 2024-11-25T12:05:15,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742205_1381 (size=12595) 2024-11-25T12:05:16,015 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/3277de8df2aa455b95902b41087cfb97 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/3277de8df2aa455b95902b41087cfb97 2024-11-25T12:05:16,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T12:05:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:16,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:16,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:16,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:16,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T12:05:16,065 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/B of 58c2c29070f1735a85b3dd1edce5016e into 3277de8df2aa455b95902b41087cfb97(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:16,065 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:16,065 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/B, priority=13, startTime=1732536315636; duration=0sec 2024-11-25T12:05:16,065 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:16,065 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:B 2024-11-25T12:05:16,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:16,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536376075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:16,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:16,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:16,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:16,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T12:05:16,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:16,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:16,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:16,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:16,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:16,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:16,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536376207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:16,227 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:16,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:16,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:16,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:16,249 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112510b7a7d1251d402f8a8cf70a2b635e41_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112510b7a7d1251d402f8a8cf70a2b635e41_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:16,253 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/87e0b99b1572441e832631000032cdaa, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:16,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/87e0b99b1572441e832631000032cdaa is 175, key is test_row_0/A:col10/1732536315112/Put/seqid=0 2024-11-25T12:05:16,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742206_1382 (size=39749) 2024-11-25T12:05:16,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T12:05:16,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:16,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:16,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:16,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:16,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:16,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:16,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536376448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:16,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:16,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T12:05:16,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:16,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:16,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:16,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:16,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:16,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T12:05:16,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:16,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:16,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:16,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:16,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:16,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:16,717 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/87e0b99b1572441e832631000032cdaa 2024-11-25T12:05:16,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/5235265fb37f4a86b69285da1e1b95db is 50, key is test_row_0/B:col10/1732536315112/Put/seqid=0 2024-11-25T12:05:16,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742207_1383 (size=12151) 2024-11-25T12:05:16,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/5235265fb37f4a86b69285da1e1b95db 2024-11-25T12:05:16,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536376789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:16,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T12:05:16,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:16,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:16,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:16,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:16,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:16,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:16,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/bbc964f960634ce6a4a1244c45d2a079 is 50, key is test_row_0/C:col10/1732536315112/Put/seqid=0 2024-11-25T12:05:16,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742208_1384 (size=12151) 2024-11-25T12:05:16,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/bbc964f960634ce6a4a1244c45d2a079 2024-11-25T12:05:16,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/87e0b99b1572441e832631000032cdaa as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/87e0b99b1572441e832631000032cdaa 2024-11-25T12:05:16,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/87e0b99b1572441e832631000032cdaa, entries=200, sequenceid=206, filesize=38.8 K 2024-11-25T12:05:16,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/5235265fb37f4a86b69285da1e1b95db as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/5235265fb37f4a86b69285da1e1b95db 2024-11-25T12:05:17,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T12:05:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:17,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:17,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:17,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:17,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/5235265fb37f4a86b69285da1e1b95db, entries=150, sequenceid=206, filesize=11.9 K 2024-11-25T12:05:17,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/bbc964f960634ce6a4a1244c45d2a079 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/bbc964f960634ce6a4a1244c45d2a079 2024-11-25T12:05:17,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/bbc964f960634ce6a4a1244c45d2a079, entries=150, sequenceid=206, filesize=11.9 K 2024-11-25T12:05:17,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 58c2c29070f1735a85b3dd1edce5016e in 1348ms, sequenceid=206, compaction requested=false 2024-11-25T12:05:17,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:17,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T12:05:17,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-25T12:05:17,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:17,185 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-25T12:05:17,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125cd51266224b1433e83c0c056cc297900_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_0/A:col10/1732536315961/Put/seqid=0 2024-11-25T12:05:17,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742209_1385 (size=12304) 2024-11-25T12:05:17,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:17,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:17,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536377421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:17,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:17,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536377425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:17,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:17,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536377429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:17,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:17,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536377432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:17,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536377444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:17,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536377557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:17,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:17,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536377557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:17,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:17,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536377569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:17,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:17,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536377569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:17,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:17,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536377580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:17,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:17,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:17,816 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125cd51266224b1433e83c0c056cc297900_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cd51266224b1433e83c0c056cc297900_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:17,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:17,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536377797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:17,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:17,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536377797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:17,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/8d7a8680bc9a401c9c3b31cd856e3109, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:17,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/8d7a8680bc9a401c9c3b31cd856e3109 is 175, key is test_row_0/A:col10/1732536315961/Put/seqid=0 2024-11-25T12:05:17,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536377800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:17,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536377800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:17,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536377801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:17,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742210_1386 (size=31105) 2024-11-25T12:05:17,869 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/8d7a8680bc9a401c9c3b31cd856e3109 2024-11-25T12:05:17,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/c890cf6f34504148826a8abb95e937e6 is 50, key is test_row_0/B:col10/1732536315961/Put/seqid=0 2024-11-25T12:05:17,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742211_1387 (size=12151) 2024-11-25T12:05:18,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536378136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:18,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536378137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:18,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:18,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:18,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536378140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:18,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536378140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:18,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:18,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536378141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:18,376 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/c890cf6f34504148826a8abb95e937e6 2024-11-25T12:05:18,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/79cc02f5753a4392ba199ef35ca338dd is 50, key is test_row_0/C:col10/1732536315961/Put/seqid=0 2024-11-25T12:05:18,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742212_1388 (size=12151) 2024-11-25T12:05:18,461 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/79cc02f5753a4392ba199ef35ca338dd 2024-11-25T12:05:18,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/8d7a8680bc9a401c9c3b31cd856e3109 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/8d7a8680bc9a401c9c3b31cd856e3109 2024-11-25T12:05:18,481 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/8d7a8680bc9a401c9c3b31cd856e3109, entries=150, sequenceid=226, filesize=30.4 K 2024-11-25T12:05:18,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/c890cf6f34504148826a8abb95e937e6 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/c890cf6f34504148826a8abb95e937e6 2024-11-25T12:05:18,491 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/c890cf6f34504148826a8abb95e937e6, entries=150, sequenceid=226, filesize=11.9 K 2024-11-25T12:05:18,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/79cc02f5753a4392ba199ef35ca338dd as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/79cc02f5753a4392ba199ef35ca338dd 2024-11-25T12:05:18,497 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/79cc02f5753a4392ba199ef35ca338dd, entries=150, sequenceid=226, filesize=11.9 K 2024-11-25T12:05:18,505 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 58c2c29070f1735a85b3dd1edce5016e in 1320ms, sequenceid=226, compaction requested=true 2024-11-25T12:05:18,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:18,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:18,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-25T12:05:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-25T12:05:18,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-25T12:05:18,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6040 sec 2024-11-25T12:05:18,529 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 3.6110 sec 2024-11-25T12:05:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:18,683 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:18,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-25T12:05:18,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256a4fc376bff94435b244364ec3375531_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_0/A:col10/1732536318676/Put/seqid=0 2024-11-25T12:05:18,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742213_1389 (size=14794) 2024-11-25T12:05:18,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536378737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:18,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:18,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536378746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:18,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536378748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:18,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536378764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:18,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536378766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:18,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:18,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:18,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536378876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:18,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:18,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536378908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:18,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536378908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:18,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:18,932 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536378908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:18,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:18,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536378909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:18,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-25T12:05:19,081 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:19,085 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:05:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-25T12:05:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T12:05:19,087 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:05:19,087 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:05:19,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:05:19,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536379117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:19,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536379136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:19,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536379136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:19,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:19,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536379136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:19,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:19,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:19,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:19,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:19,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:19,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536379136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:19,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:19,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:19,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:19,177 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T12:05:19,201 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411256a4fc376bff94435b244364ec3375531_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256a4fc376bff94435b244364ec3375531_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:19,212 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/84d49c9179ab404298b759280d42cb81, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:19,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/84d49c9179ab404298b759280d42cb81 is 175, key is test_row_0/A:col10/1732536318676/Put/seqid=0 2024-11-25T12:05:19,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-25T12:05:19,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:19,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:19,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:19,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:19,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:19,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:19,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742214_1390 (size=39749) 2024-11-25T12:05:19,279 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/84d49c9179ab404298b759280d42cb81 2024-11-25T12:05:19,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/cd4f44a3e7b546d1b0f7edbb0ef1b2b9 is 50, key is test_row_0/B:col10/1732536318676/Put/seqid=0 2024-11-25T12:05:19,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742215_1391 (size=12151) 2024-11-25T12:05:19,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/cd4f44a3e7b546d1b0f7edbb0ef1b2b9 2024-11-25T12:05:19,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/499ebb61407b4e3592d82a711fe97992 is 50, key is test_row_0/C:col10/1732536318676/Put/seqid=0 2024-11-25T12:05:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T12:05:19,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-25T12:05:19,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:19,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:19,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:19,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:19,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:19,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742216_1392 (size=12151) 2024-11-25T12:05:19,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536379457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:19,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536379457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:19,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536379465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:19,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536379465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:19,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536379465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:19,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-25T12:05:19,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:19,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:19,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:19,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:19,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:19,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:19,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T12:05:19,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-25T12:05:19,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:19,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:19,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:19,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:19,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:19,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:19,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/499ebb61407b4e3592d82a711fe97992 2024-11-25T12:05:19,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/84d49c9179ab404298b759280d42cb81 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/84d49c9179ab404298b759280d42cb81 2024-11-25T12:05:19,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/84d49c9179ab404298b759280d42cb81, entries=200, sequenceid=247, filesize=38.8 K 2024-11-25T12:05:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/cd4f44a3e7b546d1b0f7edbb0ef1b2b9 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/cd4f44a3e7b546d1b0f7edbb0ef1b2b9 2024-11-25T12:05:19,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/cd4f44a3e7b546d1b0f7edbb0ef1b2b9, entries=150, sequenceid=247, filesize=11.9 K 2024-11-25T12:05:19,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/499ebb61407b4e3592d82a711fe97992 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/499ebb61407b4e3592d82a711fe97992 2024-11-25T12:05:19,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/499ebb61407b4e3592d82a711fe97992, entries=150, sequenceid=247, filesize=11.9 K 2024-11-25T12:05:19,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 58c2c29070f1735a85b3dd1edce5016e in 1195ms, sequenceid=247, compaction requested=true 2024-11-25T12:05:19,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:19,879 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:19,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:05:19,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:19,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:05:19,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:19,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:19,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:19,880 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:19,882 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142152 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:19,882 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/A is initiating minor compaction (all files) 2024-11-25T12:05:19,882 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/A in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:19,882 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/2b0f1ff29b544035a7705e8bf8428d58, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/87e0b99b1572441e832631000032cdaa, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/8d7a8680bc9a401c9c3b31cd856e3109, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/84d49c9179ab404298b759280d42cb81] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=138.8 K 2024-11-25T12:05:19,882 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:19,882 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/2b0f1ff29b544035a7705e8bf8428d58, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/87e0b99b1572441e832631000032cdaa, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/8d7a8680bc9a401c9c3b31cd856e3109, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/84d49c9179ab404298b759280d42cb81] 2024-11-25T12:05:19,882 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:19,882 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/B is initiating minor compaction (all files) 2024-11-25T12:05:19,883 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/B in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:19,883 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/3277de8df2aa455b95902b41087cfb97, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/5235265fb37f4a86b69285da1e1b95db, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/c890cf6f34504148826a8abb95e937e6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/cd4f44a3e7b546d1b0f7edbb0ef1b2b9] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=47.9 K 2024-11-25T12:05:19,883 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b0f1ff29b544035a7705e8bf8428d58, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732536314828 2024-11-25T12:05:19,883 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 3277de8df2aa455b95902b41087cfb97, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732536314828 2024-11-25T12:05:19,883 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87e0b99b1572441e832631000032cdaa, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732536315112 2024-11-25T12:05:19,884 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 5235265fb37f4a86b69285da1e1b95db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732536315112 2024-11-25T12:05:19,884 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d7a8680bc9a401c9c3b31cd856e3109, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732536315961 2024-11-25T12:05:19,884 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting c890cf6f34504148826a8abb95e937e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732536315961 2024-11-25T12:05:19,884 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84d49c9179ab404298b759280d42cb81, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732536317418 2024-11-25T12:05:19,885 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting cd4f44a3e7b546d1b0f7edbb0ef1b2b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732536317418 2024-11-25T12:05:19,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,909 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,921 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#B#compaction#328 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:19,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,921 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/f472515455e141e8827da29a19428d9e is 50, key is test_row_0/B:col10/1732536318676/Put/seqid=0 2024-11-25T12:05:19,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,929 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125ed417105adef4ac4b67fc1eb402b5cdc_58c2c29070f1735a85b3dd1edce5016e store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,931 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125ed417105adef4ac4b67fc1eb402b5cdc_58c2c29070f1735a85b3dd1edce5016e, store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:19,932 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ed417105adef4ac4b67fc1eb402b5cdc_58c2c29070f1735a85b3dd1edce5016e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-25T12:05:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,957 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-25T12:05:19,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742217_1393 (size=12731) 2024-11-25T12:05:19,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742218_1394 (size=4469) 2024-11-25T12:05:19,991 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/f472515455e141e8827da29a19428d9e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/f472515455e141e8827da29a19428d9e 2024-11-25T12:05:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259fe6d554a024418f8cafab7ecc806367_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_1/A:col10/1732536318746/Put/seqid=0 2024-11-25T12:05:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:19,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,001 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#A#compaction#327 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:20,002 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/15799f0fc0e24ebe8bb07669f63bf6ef is 175, key is test_row_0/A:col10/1732536318676/Put/seqid=0 2024-11-25T12:05:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,007 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/B of 58c2c29070f1735a85b3dd1edce5016e into f472515455e141e8827da29a19428d9e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:20,007 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:20,007 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/B, priority=12, startTime=1732536319879; duration=0sec 2024-11-25T12:05:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,007 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:20,007 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:B 2024-11-25T12:05:20,008 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:20,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,025 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:20,025 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/C is initiating minor compaction (all files) 2024-11-25T12:05:20,025 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/C in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:20,025 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/85a77095c82d4bc9bb94f31429fe016b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/bbc964f960634ce6a4a1244c45d2a079, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/79cc02f5753a4392ba199ef35ca338dd, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/499ebb61407b4e3592d82a711fe97992] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=47.9 K 2024-11-25T12:05:20,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,029 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 85a77095c82d4bc9bb94f31429fe016b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732536314828 2024-11-25T12:05:20,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,033 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting bbc964f960634ce6a4a1244c45d2a079, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732536315112 2024-11-25T12:05:20,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,037 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 79cc02f5753a4392ba199ef35ca338dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732536315961 2024-11-25T12:05:20,041 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 499ebb61407b4e3592d82a711fe97992, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732536317418 2024-11-25T12:05:20,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742219_1395 (size=9914) 2024-11-25T12:05:20,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742220_1396 (size=31685) 2024-11-25T12:05:20,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,113 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/15799f0fc0e24ebe8bb07669f63bf6ef as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/15799f0fc0e24ebe8bb07669f63bf6ef 2024-11-25T12:05:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,119 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#C#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,119 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/019f082354d9407bbaafa86aae0d0a90 is 50, key is test_row_0/C:col10/1732536318676/Put/seqid=0 2024-11-25T12:05:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:20,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:20,136 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/A of 58c2c29070f1735a85b3dd1edce5016e into 15799f0fc0e24ebe8bb07669f63bf6ef(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:20,136 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:20,136 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/A, priority=12, startTime=1732536319879; duration=0sec 2024-11-25T12:05:20,136 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:20,136 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:A 2024-11-25T12:05:20,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742221_1397 (size=12731) 2024-11-25T12:05:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T12:05:20,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536380214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:20,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:20,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:20,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536380214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:20,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536380214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:20,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536380229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:20,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:20,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536380229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:20,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:20,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536380337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:20,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536380337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:20,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:20,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:20,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536380337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:20,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:20,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:20,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:20,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536380346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:20,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536380346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:20,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,494 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411259fe6d554a024418f8cafab7ecc806367_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259fe6d554a024418f8cafab7ecc806367_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:20,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/814e542db8d348d2a299fa58f62cb894, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:20,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/814e542db8d348d2a299fa58f62cb894 is 175, key is test_row_1/A:col10/1732536318746/Put/seqid=0 2024-11-25T12:05:20,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742222_1398 (size=22561) 2024-11-25T12:05:20,538 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/814e542db8d348d2a299fa58f62cb894 2024-11-25T12:05:20,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536380546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:20,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536380547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:20,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:20,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536380561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:20,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:20,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536380576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:20,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:20,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536380576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:20,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:20,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/88bfd6bf12614a8ca5a20a0cd701b81b is 50, key is test_row_1/B:col10/1732536318746/Put/seqid=0 2024-11-25T12:05:20,616 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/019f082354d9407bbaafa86aae0d0a90 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/019f082354d9407bbaafa86aae0d0a90 2024-11-25T12:05:20,641 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/C of 58c2c29070f1735a85b3dd1edce5016e into 019f082354d9407bbaafa86aae0d0a90(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:20,641 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:20,641 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/C, priority=12, startTime=1732536319880; duration=0sec 2024-11-25T12:05:20,641 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:20,641 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:C 2024-11-25T12:05:20,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742223_1399 (size=9857) 2024-11-25T12:05:20,688 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/88bfd6bf12614a8ca5a20a0cd701b81b 2024-11-25T12:05:20,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/9a0073b7ad7649819b32ac1e948b770e is 50, key is test_row_1/C:col10/1732536318746/Put/seqid=0 2024-11-25T12:05:20,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742224_1400 (size=9857) 2024-11-25T12:05:20,833 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/9a0073b7ad7649819b32ac1e948b770e 2024-11-25T12:05:20,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/814e542db8d348d2a299fa58f62cb894 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/814e542db8d348d2a299fa58f62cb894 2024-11-25T12:05:20,849 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/814e542db8d348d2a299fa58f62cb894, entries=100, sequenceid=262, filesize=22.0 K 2024-11-25T12:05:20,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/88bfd6bf12614a8ca5a20a0cd701b81b as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/88bfd6bf12614a8ca5a20a0cd701b81b 2024-11-25T12:05:20,861 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/88bfd6bf12614a8ca5a20a0cd701b81b, entries=100, sequenceid=262, filesize=9.6 K 2024-11-25T12:05:20,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/9a0073b7ad7649819b32ac1e948b770e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/9a0073b7ad7649819b32ac1e948b770e 2024-11-25T12:05:20,865 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/9a0073b7ad7649819b32ac1e948b770e, entries=100, sequenceid=262, filesize=9.6 K 2024-11-25T12:05:20,869 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 58c2c29070f1735a85b3dd1edce5016e in 912ms, sequenceid=262, compaction requested=false 2024-11-25T12:05:20,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:20,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:20,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-25T12:05:20,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-25T12:05:20,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-25T12:05:20,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7840 sec 2024-11-25T12:05:20,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.7890 sec 2024-11-25T12:05:20,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:20,881 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:20,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T12:05:20,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255450b60e3b8a4d42b0df8400caf078d9_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_0/A:col10/1732536320225/Put/seqid=0 2024-11-25T12:05:20,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742225_1401 (size=14994) 2024-11-25T12:05:20,946 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:20,951 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255450b60e3b8a4d42b0df8400caf078d9_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255450b60e3b8a4d42b0df8400caf078d9_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:20,952 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/42f1140400334a0695f3bacf10998bee, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:20,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/42f1140400334a0695f3bacf10998bee is 175, key is test_row_0/A:col10/1732536320225/Put/seqid=0 2024-11-25T12:05:20,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742226_1402 (size=39949) 2024-11-25T12:05:20,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536380954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:20,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:20,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:20,993 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/42f1140400334a0695f3bacf10998bee 2024-11-25T12:05:21,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536380982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:21,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536380983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:21,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536380984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:21,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:21,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536380985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:21,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:21,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:21,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/0b217ded1b8e439795f38e623420e960 is 50, key is test_row_0/B:col10/1732536320225/Put/seqid=0 2024-11-25T12:05:21,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742227_1403 (size=12301) 2024-11-25T12:05:21,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536381096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:21,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536381116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:21,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:21,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:21,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536381116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536381116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:21,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536381117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:21,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-25T12:05:21,249 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:21,250 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:05:21,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-25T12:05:21,252 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:05:21,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T12:05:21,252 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:05:21,253 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:05:21,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536381316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:21,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T12:05:21,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536381346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:21,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:21,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536381346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:21,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:21,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:21,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536381346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:21,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:21,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536381352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:21,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:21,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-25T12:05:21,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:21,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:21,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:21,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:21,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:21,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/0b217ded1b8e439795f38e623420e960 2024-11-25T12:05:21,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/831b697a2b804ccd8c598a31494e57da is 50, key is test_row_0/C:col10/1732536320225/Put/seqid=0 2024-11-25T12:05:21,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-25T12:05:21,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:21,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:21,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:21,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:21,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T12:05:21,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742228_1404 (size=12301) 2024-11-25T12:05:21,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/831b697a2b804ccd8c598a31494e57da 2024-11-25T12:05:21,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/42f1140400334a0695f3bacf10998bee as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/42f1140400334a0695f3bacf10998bee 2024-11-25T12:05:21,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536381637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:21,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/42f1140400334a0695f3bacf10998bee, entries=200, sequenceid=287, filesize=39.0 K 2024-11-25T12:05:21,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/0b217ded1b8e439795f38e623420e960 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/0b217ded1b8e439795f38e623420e960 2024-11-25T12:05:21,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/0b217ded1b8e439795f38e623420e960, entries=150, sequenceid=287, filesize=12.0 K 2024-11-25T12:05:21,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536381669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:21,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536381677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:21,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/831b697a2b804ccd8c598a31494e57da as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/831b697a2b804ccd8c598a31494e57da 2024-11-25T12:05:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:21,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:21,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536381681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:21,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:21,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536381689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:21,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:21,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:21,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-25T12:05:21,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:21,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:21,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:21,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:21,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:21,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/831b697a2b804ccd8c598a31494e57da, entries=150, sequenceid=287, filesize=12.0 K 2024-11-25T12:05:21,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 58c2c29070f1735a85b3dd1edce5016e in 858ms, sequenceid=287, compaction requested=true 2024-11-25T12:05:21,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:21,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:05:21,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:21,739 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:21,739 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:21,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:05:21,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:21,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:21,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:21,749 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:21,749 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:21,749 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/A is initiating minor compaction (all files) 2024-11-25T12:05:21,749 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/B is initiating minor compaction (all files) 2024-11-25T12:05:21,749 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/A in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:21,749 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/B in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:21,749 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/15799f0fc0e24ebe8bb07669f63bf6ef, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/814e542db8d348d2a299fa58f62cb894, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/42f1140400334a0695f3bacf10998bee] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=92.0 K 2024-11-25T12:05:21,749 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/f472515455e141e8827da29a19428d9e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/88bfd6bf12614a8ca5a20a0cd701b81b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/0b217ded1b8e439795f38e623420e960] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=34.1 K 2024-11-25T12:05:21,749 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:21,749 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/15799f0fc0e24ebe8bb07669f63bf6ef, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/814e542db8d348d2a299fa58f62cb894, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/42f1140400334a0695f3bacf10998bee] 2024-11-25T12:05:21,753 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting f472515455e141e8827da29a19428d9e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732536317418 2024-11-25T12:05:21,753 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15799f0fc0e24ebe8bb07669f63bf6ef, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732536317418 2024-11-25T12:05:21,754 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 814e542db8d348d2a299fa58f62cb894, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732536318746 2024-11-25T12:05:21,754 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 88bfd6bf12614a8ca5a20a0cd701b81b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732536318746 2024-11-25T12:05:21,761 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42f1140400334a0695f3bacf10998bee, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732536320211 2024-11-25T12:05:21,761 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b217ded1b8e439795f38e623420e960, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732536320225 2024-11-25T12:05:21,790 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#B#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:21,791 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/1b034bcdc4444e25b8e3d0bfe74835a9 is 50, key is test_row_0/B:col10/1732536320225/Put/seqid=0 2024-11-25T12:05:21,802 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:21,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742229_1405 (size=12983) 2024-11-25T12:05:21,861 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125046571bc6a0f4269b436de7426c97225_58c2c29070f1735a85b3dd1edce5016e store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:21,873 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125046571bc6a0f4269b436de7426c97225_58c2c29070f1735a85b3dd1edce5016e, store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:21,873 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125046571bc6a0f4269b436de7426c97225_58c2c29070f1735a85b3dd1edce5016e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:21,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-25T12:05:21,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:21,881 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T12:05:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T12:05:21,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742230_1406 (size=4469) 2024-11-25T12:05:21,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a9599469c55d4721be8f63ee3dfe164c_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_0/A:col10/1732536320981/Put/seqid=0 2024-11-25T12:05:22,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742231_1407 (size=12454) 2024-11-25T12:05:22,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:22,052 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125a9599469c55d4721be8f63ee3dfe164c_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a9599469c55d4721be8f63ee3dfe164c_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:22,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/61a70ba4a48a4119ab835b650f304bc9, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:22,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/61a70ba4a48a4119ab835b650f304bc9 is 175, key is test_row_0/A:col10/1732536320981/Put/seqid=0 2024-11-25T12:05:22,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742232_1408 (size=31255) 2024-11-25T12:05:22,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:22,252 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/1b034bcdc4444e25b8e3d0bfe74835a9 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/1b034bcdc4444e25b8e3d0bfe74835a9 2024-11-25T12:05:22,329 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/B of 58c2c29070f1735a85b3dd1edce5016e into 1b034bcdc4444e25b8e3d0bfe74835a9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:22,329 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:22,329 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/B, priority=13, startTime=1732536321739; duration=0sec 2024-11-25T12:05:22,329 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:22,329 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:B 2024-11-25T12:05:22,329 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:22,332 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#A#compaction#337 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:22,332 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/9d830f0cff884ebc8324b203f2d10b72 is 175, key is test_row_0/A:col10/1732536320225/Put/seqid=0 2024-11-25T12:05:22,333 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:22,333 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/C is initiating minor compaction (all files) 2024-11-25T12:05:22,333 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/C in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:22,334 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/019f082354d9407bbaafa86aae0d0a90, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/9a0073b7ad7649819b32ac1e948b770e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/831b697a2b804ccd8c598a31494e57da] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=34.1 K 2024-11-25T12:05:22,334 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 019f082354d9407bbaafa86aae0d0a90, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732536317418 2024-11-25T12:05:22,334 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a0073b7ad7649819b32ac1e948b770e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732536318746 2024-11-25T12:05:22,335 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 831b697a2b804ccd8c598a31494e57da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732536320225 2024-11-25T12:05:22,390 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#C#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:22,391 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/abf10c8419bd4f0491c2a552554641db is 50, key is test_row_0/C:col10/1732536320225/Put/seqid=0 2024-11-25T12:05:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T12:05:22,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742233_1409 (size=31937) 2024-11-25T12:05:22,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536382381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:22,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536382392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:22,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:22,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742234_1410 (size=12983) 2024-11-25T12:05:22,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536382410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:22,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536382419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:22,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536382428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:22,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:22,464 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/9d830f0cff884ebc8324b203f2d10b72 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/9d830f0cff884ebc8324b203f2d10b72 2024-11-25T12:05:22,489 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/abf10c8419bd4f0491c2a552554641db as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/abf10c8419bd4f0491c2a552554641db 2024-11-25T12:05:22,495 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/A of 58c2c29070f1735a85b3dd1edce5016e into 9d830f0cff884ebc8324b203f2d10b72(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:22,495 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:22,495 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/A, priority=13, startTime=1732536321739; duration=0sec 2024-11-25T12:05:22,495 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:22,495 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:A 2024-11-25T12:05:22,497 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=301, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/61a70ba4a48a4119ab835b650f304bc9 2024-11-25T12:05:22,514 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/C of 58c2c29070f1735a85b3dd1edce5016e into abf10c8419bd4f0491c2a552554641db(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:22,514 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:22,514 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/C, priority=13, startTime=1732536321739; duration=0sec 2024-11-25T12:05:22,514 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:22,515 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:C 2024-11-25T12:05:22,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/bd2f64634e4b4f39b10cc4b297eade69 is 50, key is test_row_0/B:col10/1732536320981/Put/seqid=0 2024-11-25T12:05:22,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536382527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:22,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536382536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:22,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742235_1411 (size=12301) 2024-11-25T12:05:22,567 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/bd2f64634e4b4f39b10cc4b297eade69 2024-11-25T12:05:22,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536382557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:22,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:22,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536382557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:22,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:22,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536382567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:22,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/53d797b3643e412b9cab2b82e7d47e32 is 50, key is test_row_0/C:col10/1732536320981/Put/seqid=0 2024-11-25T12:05:22,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742236_1412 (size=12301) 2024-11-25T12:05:22,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536382756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:22,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536382766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:22,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:22,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536382778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:22,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536382778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:22,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:22,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:22,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536382796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:22,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:22,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:23,052 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/53d797b3643e412b9cab2b82e7d47e32 2024-11-25T12:05:23,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536383077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:23,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:23,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536383089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:23,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536383096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:23,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536383096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:23,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/61a70ba4a48a4119ab835b650f304bc9 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/61a70ba4a48a4119ab835b650f304bc9 2024-11-25T12:05:23,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:23,138 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/61a70ba4a48a4119ab835b650f304bc9, entries=150, sequenceid=301, filesize=30.5 K 2024-11-25T12:05:23,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/bd2f64634e4b4f39b10cc4b297eade69 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/bd2f64634e4b4f39b10cc4b297eade69 2024-11-25T12:05:23,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536383118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:23,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:23,173 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/bd2f64634e4b4f39b10cc4b297eade69, entries=150, sequenceid=301, filesize=12.0 K 2024-11-25T12:05:23,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/53d797b3643e412b9cab2b82e7d47e32 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/53d797b3643e412b9cab2b82e7d47e32 2024-11-25T12:05:23,197 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/53d797b3643e412b9cab2b82e7d47e32, entries=150, sequenceid=301, filesize=12.0 K 2024-11-25T12:05:23,204 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 58c2c29070f1735a85b3dd1edce5016e in 1323ms, sequenceid=301, compaction requested=false 2024-11-25T12:05:23,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:23,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:23,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-25T12:05:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-25T12:05:23,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-25T12:05:23,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9750 sec 2024-11-25T12:05:23,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.0660 sec 2024-11-25T12:05:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-25T12:05:23,406 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:23,407 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:05:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-25T12:05:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T12:05:23,410 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:05:23,411 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:05:23,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:05:23,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T12:05:23,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-25T12:05:23,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:23,565 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T12:05:23,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112522eb935f62084fa6872fa1b2cdf49690_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_0/A:col10/1732536322411/Put/seqid=0 2024-11-25T12:05:23,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:23,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:23,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742237_1413 (size=12454) 2024-11-25T12:05:23,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:23,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536383667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:23,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:23,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536383677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:23,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536383681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:23,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:23,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536383689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:23,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:23,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:23,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536383692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T12:05:23,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536383796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:23,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536383806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:23,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536383806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:23,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536383821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:23,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536383827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:23,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:23,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:23,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T12:05:24,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:24,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:24,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536384029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:24,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536384029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:24,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536384032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:24,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:24,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:24,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:24,066 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112522eb935f62084fa6872fa1b2cdf49690_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112522eb935f62084fa6872fa1b2cdf49690_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:24,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/24d1d9761c0d407b931ae6cd9f4f4ab0, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:24,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/24d1d9761c0d407b931ae6cd9f4f4ab0 is 175, key is test_row_0/A:col10/1732536322411/Put/seqid=0 2024-11-25T12:05:24,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536384061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:24,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536384065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:24,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:24,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:24,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742238_1414 (size=31255) 2024-11-25T12:05:24,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536384357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:24,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536384357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:24,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:24,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:24,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:24,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:24,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:24,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:24,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:24,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536384371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:24,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:24,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:24,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:24,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536384397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:24,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536384401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:24,541 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/24d1d9761c0d407b931ae6cd9f4f4ab0 2024-11-25T12:05:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T12:05:24,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/9a8a6db0002b4261afcb66b3587810bd is 50, key is test_row_0/B:col10/1732536322411/Put/seqid=0 2024-11-25T12:05:24,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742239_1415 (size=12301) 2024-11-25T12:05:24,577 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/9a8a6db0002b4261afcb66b3587810bd 2024-11-25T12:05:24,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/085c27b83277498d8ea28f71466c4233 is 50, key is test_row_0/C:col10/1732536322411/Put/seqid=0 2024-11-25T12:05:24,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742240_1416 (size=12301) 2024-11-25T12:05:24,613 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/085c27b83277498d8ea28f71466c4233 2024-11-25T12:05:24,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/24d1d9761c0d407b931ae6cd9f4f4ab0 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/24d1d9761c0d407b931ae6cd9f4f4ab0 2024-11-25T12:05:24,631 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/24d1d9761c0d407b931ae6cd9f4f4ab0, entries=150, sequenceid=326, filesize=30.5 K 2024-11-25T12:05:24,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/9a8a6db0002b4261afcb66b3587810bd as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/9a8a6db0002b4261afcb66b3587810bd 2024-11-25T12:05:24,637 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/9a8a6db0002b4261afcb66b3587810bd, entries=150, sequenceid=326, filesize=12.0 K 2024-11-25T12:05:24,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/085c27b83277498d8ea28f71466c4233 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/085c27b83277498d8ea28f71466c4233 2024-11-25T12:05:24,642 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/085c27b83277498d8ea28f71466c4233, entries=150, sequenceid=326, filesize=12.0 K 2024-11-25T12:05:24,643 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 58c2c29070f1735a85b3dd1edce5016e in 1078ms, sequenceid=326, compaction requested=true 2024-11-25T12:05:24,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:24,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:24,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-25T12:05:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-25T12:05:24,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-25T12:05:24,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2350 sec 2024-11-25T12:05:24,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.2420 sec 2024-11-25T12:05:24,897 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:24,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-25T12:05:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:24,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255d77abdd85b54a4e821a30a7dff9f124_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_0/A:col10/1732536324895/Put/seqid=0 2024-11-25T12:05:24,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742241_1417 (size=14994) 2024-11-25T12:05:25,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536384996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:25,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536384997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:25,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:25,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536385013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:25,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:25,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536385017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:25,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:25,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536385024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:25,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:25,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:25,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536385133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536385137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:25,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536385138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:25,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:25,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536385149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:25,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:25,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536385169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:25,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:25,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:25,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536385349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:25,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536385353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:25,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536385356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:25,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:25,384 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:25,388 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255d77abdd85b54a4e821a30a7dff9f124_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255d77abdd85b54a4e821a30a7dff9f124_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:25,389 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/1a10c96dfc0245008b605a160fac6274, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:25,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/1a10c96dfc0245008b605a160fac6274 is 175, key is test_row_0/A:col10/1732536324895/Put/seqid=0 2024-11-25T12:05:25,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:25,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536385389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:25,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536385394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:25,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742242_1418 (size=39949) 2024-11-25T12:05:25,424 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=341, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/1a10c96dfc0245008b605a160fac6274 2024-11-25T12:05:25,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/e550fbba703447e396edb163096eeebd is 50, key is test_row_0/B:col10/1732536324895/Put/seqid=0 2024-11-25T12:05:25,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742243_1419 (size=12301) 2024-11-25T12:05:25,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/e550fbba703447e396edb163096eeebd 2024-11-25T12:05:25,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/4bb8dad567c84b89b048d3e902d121e6 is 50, key is test_row_0/C:col10/1732536324895/Put/seqid=0 2024-11-25T12:05:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-25T12:05:25,563 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:25,564 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:05:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-25T12:05:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T12:05:25,584 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:05:25,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742244_1420 (size=12301) 2024-11-25T12:05:25,587 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:05:25,588 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:05:25,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536385676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:25,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T12:05:25,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:25,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536385679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:25,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536385693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:25,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:25,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536385717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:25,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:25,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-25T12:05:25,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:25,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:25,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:25,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:25,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:25,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536385729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:25,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:25,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-25T12:05:25,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:25,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:25,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:25,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:25,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:25,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:25,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T12:05:25,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/4bb8dad567c84b89b048d3e902d121e6 2024-11-25T12:05:26,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/1a10c96dfc0245008b605a160fac6274 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1a10c96dfc0245008b605a160fac6274 2024-11-25T12:05:26,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1a10c96dfc0245008b605a160fac6274, entries=200, sequenceid=341, filesize=39.0 K 2024-11-25T12:05:26,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/e550fbba703447e396edb163096eeebd as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/e550fbba703447e396edb163096eeebd 2024-11-25T12:05:26,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/e550fbba703447e396edb163096eeebd, entries=150, sequenceid=341, filesize=12.0 K 2024-11-25T12:05:26,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/4bb8dad567c84b89b048d3e902d121e6 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/4bb8dad567c84b89b048d3e902d121e6 2024-11-25T12:05:26,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/4bb8dad567c84b89b048d3e902d121e6, entries=150, sequenceid=341, filesize=12.0 K 2024-11-25T12:05:26,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 58c2c29070f1735a85b3dd1edce5016e in 1125ms, sequenceid=341, compaction requested=true 2024-11-25T12:05:26,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:26,022 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:26,023 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:26,023 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/A is initiating minor compaction (all files) 2024-11-25T12:05:26,024 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/A in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:26,024 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/9d830f0cff884ebc8324b203f2d10b72, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/61a70ba4a48a4119ab835b650f304bc9, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/24d1d9761c0d407b931ae6cd9f4f4ab0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1a10c96dfc0245008b605a160fac6274] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=131.2 K 2024-11-25T12:05:26,024 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:26,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:05:26,024 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/9d830f0cff884ebc8324b203f2d10b72, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/61a70ba4a48a4119ab835b650f304bc9, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/24d1d9761c0d407b931ae6cd9f4f4ab0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1a10c96dfc0245008b605a160fac6274] 2024-11-25T12:05:26,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:26,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:05:26,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:26,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:26,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:05:26,025 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d830f0cff884ebc8324b203f2d10b72, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732536320225 2024-11-25T12:05:26,025 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61a70ba4a48a4119ab835b650f304bc9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732536320942 2024-11-25T12:05:26,026 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24d1d9761c0d407b931ae6cd9f4f4ab0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732536322377 2024-11-25T12:05:26,026 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a10c96dfc0245008b605a160fac6274, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732536323672 2024-11-25T12:05:26,028 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:26,030 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:26,030 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/B is initiating minor compaction (all files) 2024-11-25T12:05:26,030 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/B in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:26,030 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/1b034bcdc4444e25b8e3d0bfe74835a9, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/bd2f64634e4b4f39b10cc4b297eade69, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/9a8a6db0002b4261afcb66b3587810bd, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/e550fbba703447e396edb163096eeebd] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=48.7 K 2024-11-25T12:05:26,031 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b034bcdc4444e25b8e3d0bfe74835a9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732536320225 2024-11-25T12:05:26,031 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting bd2f64634e4b4f39b10cc4b297eade69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732536320942 2024-11-25T12:05:26,031 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a8a6db0002b4261afcb66b3587810bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732536322377 2024-11-25T12:05:26,032 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting e550fbba703447e396edb163096eeebd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732536323672 2024-11-25T12:05:26,037 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:26,053 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125193e80dcafee427691e825ec5bb83d13_58c2c29070f1735a85b3dd1edce5016e store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:26,056 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125193e80dcafee427691e825ec5bb83d13_58c2c29070f1735a85b3dd1edce5016e, store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:26,056 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125193e80dcafee427691e825ec5bb83d13_58c2c29070f1735a85b3dd1edce5016e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:26,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-25T12:05:26,068 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#B#compaction#349 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:26,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:26,069 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-25T12:05:26,069 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/0a92f07f06ee4a1788923b323b4f9611 is 50, key is test_row_0/B:col10/1732536324895/Put/seqid=0 2024-11-25T12:05:26,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742245_1421 (size=4469) 2024-11-25T12:05:26,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254f22234fef404c0a995a3ea990c1d14b_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_0/A:col10/1732536325021/Put/seqid=0 2024-11-25T12:05:26,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742246_1422 (size=13119) 2024-11-25T12:05:26,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742247_1423 (size=12454) 2024-11-25T12:05:26,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:26,203 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411254f22234fef404c0a995a3ea990c1d14b_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254f22234fef404c0a995a3ea990c1d14b_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:26,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/f5f290aa5d4848829eca58b7dff1c120, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:26,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/f5f290aa5d4848829eca58b7dff1c120 is 175, key is test_row_0/A:col10/1732536325021/Put/seqid=0 2024-11-25T12:05:26,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. as already flushing 2024-11-25T12:05:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T12:05:26,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742248_1424 (size=31255) 2024-11-25T12:05:26,250 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=362, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/f5f290aa5d4848829eca58b7dff1c120 2024-11-25T12:05:26,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/c24a3713170744b2a6c8803d360ce169 is 50, key is test_row_0/B:col10/1732536325021/Put/seqid=0 2024-11-25T12:05:26,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742249_1425 (size=12301) 2024-11-25T12:05:26,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536386277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:26,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:26,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536386293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:26,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536386293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:26,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:26,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536386294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:26,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536386297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:26,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:26,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:26,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536386406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:26,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:26,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536386416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:26,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536386416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:26,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536386426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:26,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:26,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536386427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:26,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:26,536 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#A#compaction#348 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:26,537 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/f85182b6faf744b8b544f00fa1daf733 is 175, key is test_row_0/A:col10/1732536324895/Put/seqid=0 2024-11-25T12:05:26,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742250_1426 (size=32073) 2024-11-25T12:05:26,613 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/0a92f07f06ee4a1788923b323b4f9611 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/0a92f07f06ee4a1788923b323b4f9611 2024-11-25T12:05:26,633 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/B of 58c2c29070f1735a85b3dd1edce5016e into 0a92f07f06ee4a1788923b323b4f9611(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:26,633 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:26,633 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/B, priority=12, startTime=1732536326024; duration=0sec 2024-11-25T12:05:26,633 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:26,633 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:B 2024-11-25T12:05:26,633 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:26,641 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:26,641 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 58c2c29070f1735a85b3dd1edce5016e/C is initiating minor compaction (all files) 2024-11-25T12:05:26,641 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 58c2c29070f1735a85b3dd1edce5016e/C in TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:26,641 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/abf10c8419bd4f0491c2a552554641db, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/53d797b3643e412b9cab2b82e7d47e32, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/085c27b83277498d8ea28f71466c4233, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/4bb8dad567c84b89b048d3e902d121e6] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp, totalSize=48.7 K 2024-11-25T12:05:26,642 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting abf10c8419bd4f0491c2a552554641db, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732536320225 2024-11-25T12:05:26,644 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 53d797b3643e412b9cab2b82e7d47e32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732536320942 2024-11-25T12:05:26,649 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 085c27b83277498d8ea28f71466c4233, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732536322377 2024-11-25T12:05:26,649 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bb8dad567c84b89b048d3e902d121e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732536323672 2024-11-25T12:05:26,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43236 deadline: 1732536386636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:26,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43192 deadline: 1732536386649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:26,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:26,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:26,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:26,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43218 deadline: 1732536386649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43178 deadline: 1732536386649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:26,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43216 deadline: 1732536386664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:26,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:26,697 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/c24a3713170744b2a6c8803d360ce169 2024-11-25T12:05:26,700 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 58c2c29070f1735a85b3dd1edce5016e#C#compaction#352 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:26,700 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/dd03558131ca45a4a44394544389dfef is 50, key is test_row_0/C:col10/1732536324895/Put/seqid=0 2024-11-25T12:05:26,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742251_1427 (size=13119) 2024-11-25T12:05:26,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/99dac128dcaa4b6da71d2659b95a6d79 is 50, key is test_row_0/C:col10/1732536325021/Put/seqid=0 2024-11-25T12:05:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T12:05:26,753 INFO [Thread-1501 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1501. 2024-11-25T12:05:26,754 DEBUG [Thread-1501 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:26,754 DEBUG [Thread-1501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:26,754 DEBUG [Thread-1501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:26,754 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:26,761 INFO [Thread-1499 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1499. 2024-11-25T12:05:26,761 DEBUG [Thread-1499 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:26,761 DEBUG [Thread-1499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:26,762 DEBUG [Thread-1499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:26,763 INFO [Thread-1495 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1495. 2024-11-25T12:05:26,764 DEBUG [Thread-1495 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:26,764 DEBUG [Thread-1495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:26,764 DEBUG [Thread-1495 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:26,764 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:26,764 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:26,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742252_1428 (size=12301) 2024-11-25T12:05:26,778 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/99dac128dcaa4b6da71d2659b95a6d79 2024-11-25T12:05:26,781 INFO [Thread-1497 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1497. 2024-11-25T12:05:26,781 DEBUG [Thread-1497 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:26,781 DEBUG [Thread-1497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:26,781 DEBUG [Thread-1497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:26,782 INFO [Thread-1503 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1503. 2024-11-25T12:05:26,782 DEBUG [Thread-1503 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:26,782 DEBUG [Thread-1503 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:26,782 DEBUG [Thread-1503 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:26,782 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:26,784 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:26,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/f5f290aa5d4848829eca58b7dff1c120 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/f5f290aa5d4848829eca58b7dff1c120 2024-11-25T12:05:26,829 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/f5f290aa5d4848829eca58b7dff1c120, entries=150, sequenceid=362, filesize=30.5 K 2024-11-25T12:05:26,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/c24a3713170744b2a6c8803d360ce169 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/c24a3713170744b2a6c8803d360ce169 2024-11-25T12:05:26,849 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/c24a3713170744b2a6c8803d360ce169, entries=150, sequenceid=362, filesize=12.0 K 2024-11-25T12:05:26,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/99dac128dcaa4b6da71d2659b95a6d79 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/99dac128dcaa4b6da71d2659b95a6d79 2024-11-25T12:05:26,870 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/99dac128dcaa4b6da71d2659b95a6d79, entries=150, sequenceid=362, filesize=12.0 K 2024-11-25T12:05:26,871 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 58c2c29070f1735a85b3dd1edce5016e in 803ms, sequenceid=362, compaction requested=false 2024-11-25T12:05:26,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:26,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:26,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-25T12:05:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-25T12:05:26,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-25T12:05:26,884 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2860 sec 2024-11-25T12:05:26,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.3200 sec 2024-11-25T12:05:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:26,989 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:26,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-25T12:05:26,990 INFO [Thread-1488 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1488. 2024-11-25T12:05:26,990 DEBUG [Thread-1488 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:26,990 DEBUG [Thread-1488 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:26,990 DEBUG [Thread-1488 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:26,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:27,004 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/f85182b6faf744b8b544f00fa1daf733 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/f85182b6faf744b8b544f00fa1daf733 2024-11-25T12:05:27,010 INFO [Thread-1486 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1486. 2024-11-25T12:05:27,010 DEBUG [Thread-1486 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:27,010 DEBUG [Thread-1486 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:27,010 DEBUG [Thread-1486 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:27,012 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:27,012 INFO [Thread-1484 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1484. 2024-11-25T12:05:27,012 DEBUG [Thread-1484 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:27,012 DEBUG [Thread-1484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:27,012 INFO [Thread-1492 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1492. 2024-11-25T12:05:27,013 DEBUG [Thread-1484 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:27,013 DEBUG [Thread-1492 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:27,013 DEBUG [Thread-1492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:27,013 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:27,013 DEBUG [Thread-1492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:27,014 INFO [Thread-1490 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1490. 2024-11-25T12:05:27,014 DEBUG [Thread-1490 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:27,014 DEBUG [Thread-1490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:27,014 DEBUG [Thread-1490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:27,014 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:27,015 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/A of 58c2c29070f1735a85b3dd1edce5016e into f85182b6faf744b8b544f00fa1daf733(size=31.3 K), total size for store is 61.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:27,015 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:27,015 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/A, priority=12, startTime=1732536326022; duration=0sec 2024-11-25T12:05:27,015 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:27,015 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:A 2024-11-25T12:05:27,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:27,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125543c8e9250084deabe7eb8dfbe535c8b_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_0/A:col10/1732536326292/Put/seqid=0 2024-11-25T12:05:27,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742253_1429 (size=12454) 2024-11-25T12:05:27,077 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:27,093 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125543c8e9250084deabe7eb8dfbe535c8b_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125543c8e9250084deabe7eb8dfbe535c8b_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:27,094 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/de84c9f4666c487e93b1bb974de90abc, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:27,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/de84c9f4666c487e93b1bb974de90abc is 175, key is test_row_0/A:col10/1732536326292/Put/seqid=0 2024-11-25T12:05:27,130 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/dd03558131ca45a4a44394544389dfef as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/dd03558131ca45a4a44394544389dfef 2024-11-25T12:05:27,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742254_1430 (size=31255) 2024-11-25T12:05:27,141 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 58c2c29070f1735a85b3dd1edce5016e/C of 58c2c29070f1735a85b3dd1edce5016e into dd03558131ca45a4a44394544389dfef(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:27,141 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:27,141 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., storeName=58c2c29070f1735a85b3dd1edce5016e/C, priority=12, startTime=1732536326024; duration=0sec 2024-11-25T12:05:27,141 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:27,141 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:C 2024-11-25T12:05:27,539 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=379, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/de84c9f4666c487e93b1bb974de90abc 2024-11-25T12:05:27,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/b3376a13ef7a4cf987d8842f52ac3641 is 50, key is test_row_0/B:col10/1732536326292/Put/seqid=0 2024-11-25T12:05:27,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742255_1431 (size=12301) 2024-11-25T12:05:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-25T12:05:27,746 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:27,746 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T12:05:27,746 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-25T12:05:27,746 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-25T12:05:27,746 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-25T12:05:27,746 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-11-25T12:05:27,746 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-25T12:05:27,747 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T12:05:27,747 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T12:05:27,747 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 539 2024-11-25T12:05:27,747 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1615 rows 2024-11-25T12:05:27,747 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 545 2024-11-25T12:05:27,747 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1628 rows 2024-11-25T12:05:27,747 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 544 2024-11-25T12:05:27,747 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1631 rows 2024-11-25T12:05:27,747 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 535 2024-11-25T12:05:27,747 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1605 rows 2024-11-25T12:05:27,747 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 531 2024-11-25T12:05:27,747 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1593 rows 2024-11-25T12:05:27,747 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T12:05:27,747 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:05:27,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:27,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:27,748 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:27,748 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-25T12:05:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T12:05:27,752 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536327752"}]},"ts":"1732536327752"} 2024-11-25T12:05:27,754 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T12:05:27,754 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T12:05:27,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T12:05:27,756 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58c2c29070f1735a85b3dd1edce5016e, UNASSIGN}] 2024-11-25T12:05:27,757 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58c2c29070f1735a85b3dd1edce5016e, UNASSIGN 2024-11-25T12:05:27,758 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=58c2c29070f1735a85b3dd1edce5016e, regionState=CLOSING, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:27,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=58c2c29070f1735a85b3dd1edce5016e, UNASSIGN because future has completed 2024-11-25T12:05:27,763 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T12:05:27,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674}] 2024-11-25T12:05:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T12:05:27,925 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:27,926 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T12:05:27,926 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing 58c2c29070f1735a85b3dd1edce5016e, disabling compactions & flushes 2024-11-25T12:05:27,926 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:27,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/b3376a13ef7a4cf987d8842f52ac3641 2024-11-25T12:05:27,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/6e6c37a4e45948c28de486b711cc54ff is 50, key is test_row_0/C:col10/1732536326292/Put/seqid=0 2024-11-25T12:05:28,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742256_1432 (size=12301) 2024-11-25T12:05:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T12:05:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T12:05:28,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/6e6c37a4e45948c28de486b711cc54ff 2024-11-25T12:05:28,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/de84c9f4666c487e93b1bb974de90abc as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/de84c9f4666c487e93b1bb974de90abc 2024-11-25T12:05:28,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/de84c9f4666c487e93b1bb974de90abc, entries=150, sequenceid=379, filesize=30.5 K 2024-11-25T12:05:28,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/b3376a13ef7a4cf987d8842f52ac3641 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/b3376a13ef7a4cf987d8842f52ac3641 2024-11-25T12:05:28,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/b3376a13ef7a4cf987d8842f52ac3641, entries=150, sequenceid=379, filesize=12.0 K 2024-11-25T12:05:28,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/6e6c37a4e45948c28de486b711cc54ff as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/6e6c37a4e45948c28de486b711cc54ff 2024-11-25T12:05:28,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/6e6c37a4e45948c28de486b711cc54ff, entries=150, sequenceid=379, filesize=12.0 K 2024-11-25T12:05:28,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=26.84 KB/27480 for 58c2c29070f1735a85b3dd1edce5016e in 1449ms, sequenceid=379, compaction requested=true 2024-11-25T12:05:28,438 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:28,438 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:28,438 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. after waiting 0 ms 2024-11-25T12:05:28,438 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:28,438 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing 58c2c29070f1735a85b3dd1edce5016e 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-25T12:05:28,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 58c2c29070f1735a85b3dd1edce5016e: 2024-11-25T12:05:28,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:05:28,440 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. because compaction request was cancelled 2024-11-25T12:05:28,440 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:A 2024-11-25T12:05:28,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:28,440 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. because compaction request was cancelled 2024-11-25T12:05:28,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:B, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:05:28,440 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:B 2024-11-25T12:05:28,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:28,440 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. because compaction request was cancelled 2024-11-25T12:05:28,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 58c2c29070f1735a85b3dd1edce5016e:C, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:05:28,440 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 58c2c29070f1735a85b3dd1edce5016e:C 2024-11-25T12:05:28,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:28,451 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250c0f5500919344aeb7b20ac4dbbbb8ff_58c2c29070f1735a85b3dd1edce5016e is 50, key is test_row_0/A:col10/1732536327001/Put/seqid=0 2024-11-25T12:05:28,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742257_1433 (size=12454) 2024-11-25T12:05:28,492 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:28,504 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411250c0f5500919344aeb7b20ac4dbbbb8ff_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250c0f5500919344aeb7b20ac4dbbbb8ff_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:28,505 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/409bfbd70593456fb96a15458effbbb6, store: [table=TestAcidGuarantees family=A region=58c2c29070f1735a85b3dd1edce5016e] 2024-11-25T12:05:28,506 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/409bfbd70593456fb96a15458effbbb6 is 175, key is test_row_0/A:col10/1732536327001/Put/seqid=0 2024-11-25T12:05:28,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742258_1434 (size=31255) 2024-11-25T12:05:28,524 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=388, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/409bfbd70593456fb96a15458effbbb6 2024-11-25T12:05:28,535 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/3193f6f5aac646ef915189eab19b6bae is 50, key is test_row_0/B:col10/1732536327001/Put/seqid=0 2024-11-25T12:05:28,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742259_1435 (size=12301) 2024-11-25T12:05:28,548 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/3193f6f5aac646ef915189eab19b6bae 2024-11-25T12:05:28,570 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/cd684734f60d4654b85e715c06f9312f is 50, key is test_row_0/C:col10/1732536327001/Put/seqid=0 2024-11-25T12:05:28,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742260_1436 (size=12301) 2024-11-25T12:05:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T12:05:28,984 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/cd684734f60d4654b85e715c06f9312f 2024-11-25T12:05:28,988 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/A/409bfbd70593456fb96a15458effbbb6 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/409bfbd70593456fb96a15458effbbb6 2024-11-25T12:05:28,992 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/409bfbd70593456fb96a15458effbbb6, entries=150, sequenceid=388, filesize=30.5 K 2024-11-25T12:05:28,993 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/B/3193f6f5aac646ef915189eab19b6bae as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/3193f6f5aac646ef915189eab19b6bae 2024-11-25T12:05:28,999 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/3193f6f5aac646ef915189eab19b6bae, entries=150, sequenceid=388, filesize=12.0 K 2024-11-25T12:05:29,000 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/.tmp/C/cd684734f60d4654b85e715c06f9312f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/cd684734f60d4654b85e715c06f9312f 2024-11-25T12:05:29,004 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/cd684734f60d4654b85e715c06f9312f, entries=150, sequenceid=388, filesize=12.0 K 2024-11-25T12:05:29,009 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 58c2c29070f1735a85b3dd1edce5016e in 571ms, sequenceid=388, compaction requested=true 2024-11-25T12:05:29,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/963175be125e4c1aac430eeecb40ff76, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/0f80a948cae147968af4b9d38c997115, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/580288d0d0064e3ca0f5769d42d051ea, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/0d203ab2031444959a295609cbbacd47, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/76469d48e546472a893f28ca783d591b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/710147d1ac8e4205a4d33028aedc77e2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/5e30dced5cab47eab4cd5769d1b56b85, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1db1cbe49caf42d4bf2abe3d4814e52a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/c0cd8460afed4b99953029724d68933a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/3ef16a6a178942d5b9b62f78e116022a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/45c9ffb5ff5b482fa047eefd89af330b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1aa5ade524704c048fc41229e129d141, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/2421e91783f346be9a89df78980f4a2c, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/2b0f1ff29b544035a7705e8bf8428d58, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/87e0b99b1572441e832631000032cdaa, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/8d7a8680bc9a401c9c3b31cd856e3109, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/84d49c9179ab404298b759280d42cb81, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/15799f0fc0e24ebe8bb07669f63bf6ef, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/814e542db8d348d2a299fa58f62cb894, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/42f1140400334a0695f3bacf10998bee, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/9d830f0cff884ebc8324b203f2d10b72, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/61a70ba4a48a4119ab835b650f304bc9, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/24d1d9761c0d407b931ae6cd9f4f4ab0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1a10c96dfc0245008b605a160fac6274] to archive 2024-11-25T12:05:29,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:05:29,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/963175be125e4c1aac430eeecb40ff76 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/963175be125e4c1aac430eeecb40ff76 2024-11-25T12:05:29,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/0f80a948cae147968af4b9d38c997115 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/0f80a948cae147968af4b9d38c997115 2024-11-25T12:05:29,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/580288d0d0064e3ca0f5769d42d051ea to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/580288d0d0064e3ca0f5769d42d051ea 2024-11-25T12:05:29,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/0d203ab2031444959a295609cbbacd47 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/0d203ab2031444959a295609cbbacd47 2024-11-25T12:05:29,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/76469d48e546472a893f28ca783d591b to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/76469d48e546472a893f28ca783d591b 2024-11-25T12:05:29,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/710147d1ac8e4205a4d33028aedc77e2 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/710147d1ac8e4205a4d33028aedc77e2 2024-11-25T12:05:29,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/5e30dced5cab47eab4cd5769d1b56b85 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/5e30dced5cab47eab4cd5769d1b56b85 2024-11-25T12:05:29,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1db1cbe49caf42d4bf2abe3d4814e52a to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1db1cbe49caf42d4bf2abe3d4814e52a 2024-11-25T12:05:29,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/c0cd8460afed4b99953029724d68933a to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/c0cd8460afed4b99953029724d68933a 2024-11-25T12:05:29,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/3ef16a6a178942d5b9b62f78e116022a to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/3ef16a6a178942d5b9b62f78e116022a 2024-11-25T12:05:29,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/45c9ffb5ff5b482fa047eefd89af330b to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/45c9ffb5ff5b482fa047eefd89af330b 2024-11-25T12:05:29,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1aa5ade524704c048fc41229e129d141 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1aa5ade524704c048fc41229e129d141 2024-11-25T12:05:29,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/2421e91783f346be9a89df78980f4a2c to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/2421e91783f346be9a89df78980f4a2c 2024-11-25T12:05:29,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/2b0f1ff29b544035a7705e8bf8428d58 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/2b0f1ff29b544035a7705e8bf8428d58 2024-11-25T12:05:29,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/87e0b99b1572441e832631000032cdaa to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/87e0b99b1572441e832631000032cdaa 2024-11-25T12:05:29,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/8d7a8680bc9a401c9c3b31cd856e3109 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/8d7a8680bc9a401c9c3b31cd856e3109 2024-11-25T12:05:29,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/84d49c9179ab404298b759280d42cb81 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/84d49c9179ab404298b759280d42cb81 2024-11-25T12:05:29,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/15799f0fc0e24ebe8bb07669f63bf6ef to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/15799f0fc0e24ebe8bb07669f63bf6ef 2024-11-25T12:05:29,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/814e542db8d348d2a299fa58f62cb894 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/814e542db8d348d2a299fa58f62cb894 2024-11-25T12:05:29,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/42f1140400334a0695f3bacf10998bee to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/42f1140400334a0695f3bacf10998bee 2024-11-25T12:05:29,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/9d830f0cff884ebc8324b203f2d10b72 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/9d830f0cff884ebc8324b203f2d10b72 2024-11-25T12:05:29,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/61a70ba4a48a4119ab835b650f304bc9 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/61a70ba4a48a4119ab835b650f304bc9 2024-11-25T12:05:29,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/24d1d9761c0d407b931ae6cd9f4f4ab0 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/24d1d9761c0d407b931ae6cd9f4f4ab0 2024-11-25T12:05:29,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1a10c96dfc0245008b605a160fac6274 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/1a10c96dfc0245008b605a160fac6274 2024-11-25T12:05:29,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/8512e73d0d3345e7a6c88c2d010a7426, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/1b2f7ad2bf874076b786bda35e5f3eb9, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/583e04ec00c34cca95a29dc2ea92150c, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/ab99aa6a16c3430f836d5c46df28b889, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/3edd7b51ff2f45b6a4ca09082a50b731, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/7103b08098c14422ad6954d8ef00ceba, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/f7ed7bef6a2d446eabaa8a912cf00727, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/881c5b0ead624a9e9ca0d7a675594658, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/c6d7ced7008945b88fa8154cbb031398, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/3b806a1c72ad4e6d83e92c641ae4272e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/436cf7930a7a4c7bbc19d84487427fb7, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/2293763114b349e493bbc87476674191, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/3277de8df2aa455b95902b41087cfb97, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/ce1d5aec2aee45a2bbbd7ba633756ab1, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/5235265fb37f4a86b69285da1e1b95db, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/c890cf6f34504148826a8abb95e937e6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/f472515455e141e8827da29a19428d9e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/cd4f44a3e7b546d1b0f7edbb0ef1b2b9, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/88bfd6bf12614a8ca5a20a0cd701b81b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/1b034bcdc4444e25b8e3d0bfe74835a9, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/0b217ded1b8e439795f38e623420e960, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/bd2f64634e4b4f39b10cc4b297eade69, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/9a8a6db0002b4261afcb66b3587810bd, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/e550fbba703447e396edb163096eeebd] to archive 2024-11-25T12:05:29,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:05:29,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/8512e73d0d3345e7a6c88c2d010a7426 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/8512e73d0d3345e7a6c88c2d010a7426 2024-11-25T12:05:29,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/1b2f7ad2bf874076b786bda35e5f3eb9 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/1b2f7ad2bf874076b786bda35e5f3eb9 2024-11-25T12:05:29,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/583e04ec00c34cca95a29dc2ea92150c to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/583e04ec00c34cca95a29dc2ea92150c 2024-11-25T12:05:29,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/ab99aa6a16c3430f836d5c46df28b889 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/ab99aa6a16c3430f836d5c46df28b889 2024-11-25T12:05:29,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/3edd7b51ff2f45b6a4ca09082a50b731 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/3edd7b51ff2f45b6a4ca09082a50b731 2024-11-25T12:05:29,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/7103b08098c14422ad6954d8ef00ceba to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/7103b08098c14422ad6954d8ef00ceba 2024-11-25T12:05:29,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/f7ed7bef6a2d446eabaa8a912cf00727 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/f7ed7bef6a2d446eabaa8a912cf00727 2024-11-25T12:05:29,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/881c5b0ead624a9e9ca0d7a675594658 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/881c5b0ead624a9e9ca0d7a675594658 2024-11-25T12:05:29,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/c6d7ced7008945b88fa8154cbb031398 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/c6d7ced7008945b88fa8154cbb031398 2024-11-25T12:05:29,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/3b806a1c72ad4e6d83e92c641ae4272e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/3b806a1c72ad4e6d83e92c641ae4272e 2024-11-25T12:05:29,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/436cf7930a7a4c7bbc19d84487427fb7 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/436cf7930a7a4c7bbc19d84487427fb7 2024-11-25T12:05:29,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/2293763114b349e493bbc87476674191 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/2293763114b349e493bbc87476674191 2024-11-25T12:05:29,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/3277de8df2aa455b95902b41087cfb97 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/3277de8df2aa455b95902b41087cfb97 2024-11-25T12:05:29,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/ce1d5aec2aee45a2bbbd7ba633756ab1 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/ce1d5aec2aee45a2bbbd7ba633756ab1 2024-11-25T12:05:29,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/5235265fb37f4a86b69285da1e1b95db to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/5235265fb37f4a86b69285da1e1b95db 2024-11-25T12:05:29,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/c890cf6f34504148826a8abb95e937e6 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/c890cf6f34504148826a8abb95e937e6 2024-11-25T12:05:29,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/f472515455e141e8827da29a19428d9e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/f472515455e141e8827da29a19428d9e 2024-11-25T12:05:29,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/cd4f44a3e7b546d1b0f7edbb0ef1b2b9 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/cd4f44a3e7b546d1b0f7edbb0ef1b2b9 2024-11-25T12:05:29,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/88bfd6bf12614a8ca5a20a0cd701b81b to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/88bfd6bf12614a8ca5a20a0cd701b81b 2024-11-25T12:05:29,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/1b034bcdc4444e25b8e3d0bfe74835a9 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/1b034bcdc4444e25b8e3d0bfe74835a9 2024-11-25T12:05:29,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/0b217ded1b8e439795f38e623420e960 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/0b217ded1b8e439795f38e623420e960 2024-11-25T12:05:29,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/bd2f64634e4b4f39b10cc4b297eade69 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/bd2f64634e4b4f39b10cc4b297eade69 2024-11-25T12:05:29,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/9a8a6db0002b4261afcb66b3587810bd to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/9a8a6db0002b4261afcb66b3587810bd 2024-11-25T12:05:29,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/e550fbba703447e396edb163096eeebd to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/e550fbba703447e396edb163096eeebd 2024-11-25T12:05:29,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/6c868e90be714b25863e7da9d1ccdb43, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/140f6114020a4796ab0f819d9c146b53, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/03d4935112b94db49ddec8adf54d1bf0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/d2ac38eeae8d4a08a93a38ac6eb10934, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/c0ab62b235c64fb9a25ccfe5085bf081, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/85e330c957d34b56a6c33c376a5788f1, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/3143cd2f06964a3aa249c8c46f767009, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/de528d79cf5547d8a4762d6f7f1a8b24, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/16a6d5b858014b9e8a813d6d31dd6434, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/36b5c5f2389d4cd0a6ccc908424c5d8e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/164566bd72e44897a7df19bbc97c9121, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/1c22c029fb7e4f0e835b4a61a8ddb2ba, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/85a77095c82d4bc9bb94f31429fe016b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/2fb35d656c33423eb26bb4e13211df0a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/bbc964f960634ce6a4a1244c45d2a079, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/79cc02f5753a4392ba199ef35ca338dd, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/019f082354d9407bbaafa86aae0d0a90, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/499ebb61407b4e3592d82a711fe97992, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/9a0073b7ad7649819b32ac1e948b770e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/abf10c8419bd4f0491c2a552554641db, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/831b697a2b804ccd8c598a31494e57da, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/53d797b3643e412b9cab2b82e7d47e32, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/085c27b83277498d8ea28f71466c4233, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/4bb8dad567c84b89b048d3e902d121e6] to archive 2024-11-25T12:05:29,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:05:29,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/6c868e90be714b25863e7da9d1ccdb43 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/6c868e90be714b25863e7da9d1ccdb43 2024-11-25T12:05:29,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/140f6114020a4796ab0f819d9c146b53 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/140f6114020a4796ab0f819d9c146b53 2024-11-25T12:05:29,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/03d4935112b94db49ddec8adf54d1bf0 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/03d4935112b94db49ddec8adf54d1bf0 2024-11-25T12:05:29,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/d2ac38eeae8d4a08a93a38ac6eb10934 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/d2ac38eeae8d4a08a93a38ac6eb10934 2024-11-25T12:05:29,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/c0ab62b235c64fb9a25ccfe5085bf081 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/c0ab62b235c64fb9a25ccfe5085bf081 2024-11-25T12:05:29,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/85e330c957d34b56a6c33c376a5788f1 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/85e330c957d34b56a6c33c376a5788f1 2024-11-25T12:05:29,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/3143cd2f06964a3aa249c8c46f767009 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/3143cd2f06964a3aa249c8c46f767009 2024-11-25T12:05:29,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/de528d79cf5547d8a4762d6f7f1a8b24 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/de528d79cf5547d8a4762d6f7f1a8b24 2024-11-25T12:05:29,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/16a6d5b858014b9e8a813d6d31dd6434 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/16a6d5b858014b9e8a813d6d31dd6434 2024-11-25T12:05:29,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/36b5c5f2389d4cd0a6ccc908424c5d8e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/36b5c5f2389d4cd0a6ccc908424c5d8e 2024-11-25T12:05:29,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/164566bd72e44897a7df19bbc97c9121 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/164566bd72e44897a7df19bbc97c9121 2024-11-25T12:05:29,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/1c22c029fb7e4f0e835b4a61a8ddb2ba to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/1c22c029fb7e4f0e835b4a61a8ddb2ba 2024-11-25T12:05:29,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/85a77095c82d4bc9bb94f31429fe016b to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/85a77095c82d4bc9bb94f31429fe016b 2024-11-25T12:05:29,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/2fb35d656c33423eb26bb4e13211df0a to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/2fb35d656c33423eb26bb4e13211df0a 2024-11-25T12:05:29,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/bbc964f960634ce6a4a1244c45d2a079 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/bbc964f960634ce6a4a1244c45d2a079 2024-11-25T12:05:29,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/79cc02f5753a4392ba199ef35ca338dd to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/79cc02f5753a4392ba199ef35ca338dd 2024-11-25T12:05:29,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/019f082354d9407bbaafa86aae0d0a90 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/019f082354d9407bbaafa86aae0d0a90 2024-11-25T12:05:29,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/499ebb61407b4e3592d82a711fe97992 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/499ebb61407b4e3592d82a711fe97992 2024-11-25T12:05:29,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/9a0073b7ad7649819b32ac1e948b770e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/9a0073b7ad7649819b32ac1e948b770e 2024-11-25T12:05:29,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/abf10c8419bd4f0491c2a552554641db to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/abf10c8419bd4f0491c2a552554641db 2024-11-25T12:05:29,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/831b697a2b804ccd8c598a31494e57da to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/831b697a2b804ccd8c598a31494e57da 2024-11-25T12:05:29,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/53d797b3643e412b9cab2b82e7d47e32 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/53d797b3643e412b9cab2b82e7d47e32 2024-11-25T12:05:29,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/085c27b83277498d8ea28f71466c4233 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/085c27b83277498d8ea28f71466c4233 2024-11-25T12:05:29,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/4bb8dad567c84b89b048d3e902d121e6 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/4bb8dad567c84b89b048d3e902d121e6 2024-11-25T12:05:29,140 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/recovered.edits/391.seqid, newMaxSeqId=391, maxSeqId=4 2024-11-25T12:05:29,141 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. 2024-11-25T12:05:29,142 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for 58c2c29070f1735a85b3dd1edce5016e: Waiting for close lock at 1732536327926Running coprocessor pre-close hooks at 1732536327926Disabling compacts and flushes for region at 1732536327926Disabling writes for close at 1732536328438 (+512 ms)Obtaining lock to block concurrent updates at 1732536328438Preparing flush snapshotting stores in 58c2c29070f1735a85b3dd1edce5016e at 1732536328438Finished memstore snapshotting TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732536328439 (+1 ms)Flushing stores of TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e. at 1732536328439Flushing 58c2c29070f1735a85b3dd1edce5016e/A: creating writer at 1732536328439Flushing mob file 58c2c29070f1735a85b3dd1edce5016e/A: appending metadata at 1732536328451 (+12 ms)Flushing mob file 58c2c29070f1735a85b3dd1edce5016e/A: closing flushed file at 1732536328451Flushing 58c2c29070f1735a85b3dd1edce5016e/A: appending metadata at 1732536328505 (+54 ms)Flushing 58c2c29070f1735a85b3dd1edce5016e/A: closing flushed file at 1732536328505Flushing 58c2c29070f1735a85b3dd1edce5016e/B: creating writer at 1732536328531 (+26 ms)Flushing 58c2c29070f1735a85b3dd1edce5016e/B: appending metadata at 1732536328534 (+3 ms)Flushing 58c2c29070f1735a85b3dd1edce5016e/B: closing flushed file at 1732536328535 (+1 ms)Flushing 58c2c29070f1735a85b3dd1edce5016e/C: creating writer at 1732536328553 (+18 ms)Flushing 58c2c29070f1735a85b3dd1edce5016e/C: appending metadata at 1732536328569 (+16 ms)Flushing 58c2c29070f1735a85b3dd1edce5016e/C: closing flushed file at 1732536328570 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@499cbf4e: reopening flushed file at 1732536328988 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bda1b12: reopening flushed file at 1732536328992 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3038a4d7: reopening flushed file at 1732536329000 (+8 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 58c2c29070f1735a85b3dd1edce5016e in 571ms, sequenceid=388, compaction requested=true at 1732536329009 (+9 ms)Writing region close event to WAL at 1732536329133 (+124 ms)Running coprocessor post-close hooks at 1732536329141 (+8 ms)Closed at 1732536329141 2024-11-25T12:05:29,144 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed 58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:29,145 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=58c2c29070f1735a85b3dd1edce5016e, regionState=CLOSED 2024-11-25T12:05:29,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 because future has completed 2024-11-25T12:05:29,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-25T12:05:29,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure 58c2c29070f1735a85b3dd1edce5016e, server=2bf237fd86ca,37643,1732536214674 in 1.3880 sec 2024-11-25T12:05:29,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-25T12:05:29,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=58c2c29070f1735a85b3dd1edce5016e, UNASSIGN in 1.3970 sec 2024-11-25T12:05:29,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-25T12:05:29,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4020 sec 2024-11-25T12:05:29,161 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536329161"}]},"ts":"1732536329161"} 2024-11-25T12:05:29,163 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T12:05:29,163 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T12:05:29,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4150 sec 2024-11-25T12:05:29,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-25T12:05:29,917 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:29,918 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-25T12:05:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:29,921 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:29,922 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:29,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T12:05:29,930 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:29,934 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/recovered.edits] 2024-11-25T12:05:29,938 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/409bfbd70593456fb96a15458effbbb6 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/409bfbd70593456fb96a15458effbbb6 2024-11-25T12:05:29,941 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/de84c9f4666c487e93b1bb974de90abc to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/de84c9f4666c487e93b1bb974de90abc 2024-11-25T12:05:29,942 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/f5f290aa5d4848829eca58b7dff1c120 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/f5f290aa5d4848829eca58b7dff1c120 2024-11-25T12:05:29,943 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/f85182b6faf744b8b544f00fa1daf733 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/A/f85182b6faf744b8b544f00fa1daf733 2024-11-25T12:05:29,946 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/0a92f07f06ee4a1788923b323b4f9611 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/0a92f07f06ee4a1788923b323b4f9611 2024-11-25T12:05:29,947 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/3193f6f5aac646ef915189eab19b6bae to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/3193f6f5aac646ef915189eab19b6bae 2024-11-25T12:05:29,948 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/b3376a13ef7a4cf987d8842f52ac3641 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/b3376a13ef7a4cf987d8842f52ac3641 2024-11-25T12:05:29,950 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/c24a3713170744b2a6c8803d360ce169 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/B/c24a3713170744b2a6c8803d360ce169 2024-11-25T12:05:29,952 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/6e6c37a4e45948c28de486b711cc54ff to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/6e6c37a4e45948c28de486b711cc54ff 2024-11-25T12:05:29,954 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/99dac128dcaa4b6da71d2659b95a6d79 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/99dac128dcaa4b6da71d2659b95a6d79 2024-11-25T12:05:29,960 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/cd684734f60d4654b85e715c06f9312f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/cd684734f60d4654b85e715c06f9312f 2024-11-25T12:05:29,970 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/dd03558131ca45a4a44394544389dfef to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/C/dd03558131ca45a4a44394544389dfef 2024-11-25T12:05:29,978 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/recovered.edits/391.seqid to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e/recovered.edits/391.seqid 2024-11-25T12:05:29,979 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:29,979 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T12:05:29,980 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T12:05:29,981 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-25T12:05:29,996 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125068aa1d04df9453e96674fb0be176bc9_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125068aa1d04df9453e96674fb0be176bc9_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:29,997 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250c0f5500919344aeb7b20ac4dbbbb8ff_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411250c0f5500919344aeb7b20ac4dbbbb8ff_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:30,001 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112510586990cc764c21830469034f108e7e_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112510586990cc764c21830469034f108e7e_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:30,004 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112510b7a7d1251d402f8a8cf70a2b635e41_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112510b7a7d1251d402f8a8cf70a2b635e41_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:30,005 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112522eb935f62084fa6872fa1b2cdf49690_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112522eb935f62084fa6872fa1b2cdf49690_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:30,006 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253e31cb31c8604d1b80989f76377e2e97_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253e31cb31c8604d1b80989f76377e2e97_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:30,008 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254f22234fef404c0a995a3ea990c1d14b_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411254f22234fef404c0a995a3ea990c1d14b_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:30,011 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125543c8e9250084deabe7eb8dfbe535c8b_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125543c8e9250084deabe7eb8dfbe535c8b_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:30,012 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255450b60e3b8a4d42b0df8400caf078d9_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255450b60e3b8a4d42b0df8400caf078d9_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:30,013 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255d77abdd85b54a4e821a30a7dff9f124_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255d77abdd85b54a4e821a30a7dff9f124_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:30,014 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125699f5cf667124bee95e4db2fa1e993c9_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125699f5cf667124bee95e4db2fa1e993c9_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:30,015 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256a4fc376bff94435b244364ec3375531_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411256a4fc376bff94435b244364ec3375531_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:30,016 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112578a0e83b8d434b8686122f928ca8fe0f_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112578a0e83b8d434b8686122f928ca8fe0f_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:30,018 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257ea6f91b84904ef29304a3e9371ed1c1_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411257ea6f91b84904ef29304a3e9371ed1c1_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:30,020 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259fe6d554a024418f8cafab7ecc806367_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411259fe6d554a024418f8cafab7ecc806367_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:30,021 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a9599469c55d4721be8f63ee3dfe164c_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125a9599469c55d4721be8f63ee3dfe164c_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:30,023 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c0ecfafa9fad4ad79a0c5f370bbfcc2b_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c0ecfafa9fad4ad79a0c5f370bbfcc2b_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:30,024 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c368954a1257480b8aaaa0b55d73f1a8_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125c368954a1257480b8aaaa0b55d73f1a8_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:30,025 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ca42635f874149498fcafd29b40dc028_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ca42635f874149498fcafd29b40dc028_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:30,026 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cd51266224b1433e83c0c056cc297900_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cd51266224b1433e83c0c056cc297900_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T12:05:30,028 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125dc275074e8254961b12193e3404c9ebb_58c2c29070f1735a85b3dd1edce5016e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125dc275074e8254961b12193e3404c9ebb_58c2c29070f1735a85b3dd1edce5016e 2024-11-25T12:05:30,028 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T12:05:30,037 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:30,039 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T12:05:30,042 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T12:05:30,043 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:30,043 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T12:05:30,043 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732536330043"}]},"ts":"9223372036854775807"} 2024-11-25T12:05:30,045 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T12:05:30,045 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 58c2c29070f1735a85b3dd1edce5016e, NAME => 'TestAcidGuarantees,,1732536304896.58c2c29070f1735a85b3dd1edce5016e.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T12:05:30,045 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T12:05:30,046 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732536330045"}]},"ts":"9223372036854775807"} 2024-11-25T12:05:30,047 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T12:05:30,048 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:30,049 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 130 msec 2024-11-25T12:05:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-25T12:05:30,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T12:05:30,238 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:30,256 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 240) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2009888188_22 at /127.0.0.1:45640 [Waiting for operation #659] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-514927083_22 at /127.0.0.1:45754 [Waiting for operation #222] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2009888188_22 at /127.0.0.1:34976 [Waiting for operation #961] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-514927083_22 at /127.0.0.1:34828 [Waiting for operation #1003] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=465 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1437 (was 1459), ProcessCount=11 (was 11), AvailableMemoryMB=6645 (was 7074) 2024-11-25T12:05:30,280 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=465, MaxFileDescriptor=1048576, SystemLoadAverage=1437, ProcessCount=11, AvailableMemoryMB=6643 2024-11-25T12:05:30,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T12:05:30,283 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:05:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:30,287 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T12:05:30,287 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:30,288 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-11-25T12:05:30,288 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T12:05:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-25T12:05:30,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742261_1437 (size=959) 2024-11-25T12:05:30,306 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d 2024-11-25T12:05:30,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742262_1438 (size=53) 2024-11-25T12:05:30,319 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:05:30,319 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d2665826b05855fc4d139a82044b59e6, disabling compactions & flushes 2024-11-25T12:05:30,319 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:30,319 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:30,319 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. after waiting 0 ms 2024-11-25T12:05:30,319 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:30,319 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:30,319 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d2665826b05855fc4d139a82044b59e6: Waiting for close lock at 1732536330319Disabling compacts and flushes for region at 1732536330319Disabling writes for close at 1732536330319Writing region close event to WAL at 1732536330319Closed at 1732536330319 2024-11-25T12:05:30,320 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T12:05:30,321 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732536330320"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732536330320"}]},"ts":"1732536330320"} 2024-11-25T12:05:30,322 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T12:05:30,323 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T12:05:30,324 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536330323"}]},"ts":"1732536330323"} 2024-11-25T12:05:30,325 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T12:05:30,326 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2665826b05855fc4d139a82044b59e6, ASSIGN}] 2024-11-25T12:05:30,327 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2665826b05855fc4d139a82044b59e6, ASSIGN 2024-11-25T12:05:30,328 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2665826b05855fc4d139a82044b59e6, ASSIGN; state=OFFLINE, location=2bf237fd86ca,37643,1732536214674; forceNewPlan=false, retain=false 2024-11-25T12:05:30,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-25T12:05:30,479 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=d2665826b05855fc4d139a82044b59e6, regionState=OPENING, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:30,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2665826b05855fc4d139a82044b59e6, ASSIGN because future has completed 2024-11-25T12:05:30,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674}] 2024-11-25T12:05:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-25T12:05:30,637 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:30,637 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:05:30,637 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:30,637 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:05:30,638 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:30,638 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:30,639 INFO [StoreOpener-d2665826b05855fc4d139a82044b59e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:30,640 INFO [StoreOpener-d2665826b05855fc4d139a82044b59e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2665826b05855fc4d139a82044b59e6 columnFamilyName A 2024-11-25T12:05:30,640 DEBUG [StoreOpener-d2665826b05855fc4d139a82044b59e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:30,640 INFO [StoreOpener-d2665826b05855fc4d139a82044b59e6-1 {}] regionserver.HStore(327): Store=d2665826b05855fc4d139a82044b59e6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:05:30,640 INFO [StoreOpener-d2665826b05855fc4d139a82044b59e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:30,641 INFO [StoreOpener-d2665826b05855fc4d139a82044b59e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2665826b05855fc4d139a82044b59e6 columnFamilyName B 2024-11-25T12:05:30,641 DEBUG [StoreOpener-d2665826b05855fc4d139a82044b59e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:30,641 INFO [StoreOpener-d2665826b05855fc4d139a82044b59e6-1 {}] regionserver.HStore(327): Store=d2665826b05855fc4d139a82044b59e6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:05:30,641 INFO [StoreOpener-d2665826b05855fc4d139a82044b59e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:30,642 INFO [StoreOpener-d2665826b05855fc4d139a82044b59e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2665826b05855fc4d139a82044b59e6 columnFamilyName C 2024-11-25T12:05:30,642 DEBUG [StoreOpener-d2665826b05855fc4d139a82044b59e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:30,643 INFO [StoreOpener-d2665826b05855fc4d139a82044b59e6-1 {}] regionserver.HStore(327): Store=d2665826b05855fc4d139a82044b59e6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:05:30,643 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:30,644 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:30,644 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:30,645 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:30,645 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:30,645 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:05:30,646 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:30,648 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:05:30,648 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened d2665826b05855fc4d139a82044b59e6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71434164, jitterRate=0.06445199251174927}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:05:30,648 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:30,649 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for d2665826b05855fc4d139a82044b59e6: Running coprocessor pre-open hook at 1732536330638Writing region info on filesystem at 1732536330638Initializing all the Stores at 1732536330638Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536330638Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536330638Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536330638Cleaning up temporary data from old regions at 1732536330645 (+7 ms)Running coprocessor post-open hooks at 1732536330648 (+3 ms)Region opened successfully at 1732536330649 (+1 ms) 2024-11-25T12:05:30,649 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., pid=116, masterSystemTime=1732536330634 2024-11-25T12:05:30,651 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:30,651 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:30,651 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=d2665826b05855fc4d139a82044b59e6, regionState=OPEN, openSeqNum=2, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:30,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 because future has completed 2024-11-25T12:05:30,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-25T12:05:30,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 in 173 msec 2024-11-25T12:05:30,657 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-25T12:05:30,657 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2665826b05855fc4d139a82044b59e6, ASSIGN in 329 msec 2024-11-25T12:05:30,657 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T12:05:30,657 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536330657"}]},"ts":"1732536330657"} 2024-11-25T12:05:30,659 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T12:05:30,665 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T12:05:30,667 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 381 msec 2024-11-25T12:05:30,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-25T12:05:30,916 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:30,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d8b9445, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,918 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:30,918 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:30,919 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:30,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:30,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@409b9eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:30,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:30,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:30,921 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:30,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d5ff8dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,922 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:30,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:30,923 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:30,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:30,924 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T12:05:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 2bf237fd86ca,40019,1732536213609 2024-11-25T12:05:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7f7e187c 2024-11-25T12:05:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T12:05:30,926 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T12:05:30,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f556e7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,927 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:30,927 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:30,928 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@86ac87d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:30,929 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:30,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d51f127, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47d863fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,930 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:30,930 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:30,930 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23fc69a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:30,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:30,932 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e2948, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17cfa5a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,937 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:30,937 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:30,937 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:30,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a86036, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:30,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:30,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:30,938 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:30,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c807bfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13b94fc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,939 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:30,940 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:30,940 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a952b98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:30,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:30,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:30,941 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cd6f479, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53dea8b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,942 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:30,942 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:30,943 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f653a51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:30,944 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:30,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@673039a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,945 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3383a632, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,945 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:30,945 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:30,946 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:30,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:30,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:30,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fa6462d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:30,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:30,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:30,947 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:30,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@255a6e5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d74020e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,948 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:30,948 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:30,948 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:30,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4178ded6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:30,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:30,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:30,949 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:30,950 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ddfeb7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27adc8ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,953 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:30,953 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:30,953 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75603099, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:30,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:30,955 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:30,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a3c7664, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d3fce0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,956 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:30,956 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:30,956 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46e2adb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:30,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:30,958 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b5f6512, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,960 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337c72b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,960 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:30,961 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:30,961 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b3b9c98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:30,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:30,962 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:30,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a73240, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:30,968 DEBUG [Thread-1983 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:30,968 DEBUG [Thread-1974 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:30,968 DEBUG [Thread-1968 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:30,968 DEBUG [Thread-1970 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:30,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:30,969 DEBUG [Thread-1968 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:30,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:30,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:30,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:30,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:30,972 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:05:30,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:05:30,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:05:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:05:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-25T12:05:30,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:05:30,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T12:05:30,981 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:05:30,982 DEBUG [Thread-1981 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:30,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:30,984 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:30,985 DEBUG [Thread-1976 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:30,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:30,986 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:05:30,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:05:30,987 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:30,987 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:30,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:05:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:30,991 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:30,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T12:05:30,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:05:30,993 DEBUG [Thread-1987 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:30,993 DEBUG [Thread-1972 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:30,994 DEBUG [Thread-1979 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:30,995 DEBUG [Thread-1979 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:30,995 DEBUG [Thread-1979 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:30,996 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:30,997 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:30,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:30,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:30,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:30,999 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:31,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:31,001 DEBUG [Thread-1985 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:05:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:05:31,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:31,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:31,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:05:31,005 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:31,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2] 2024-11-25T12:05:31,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/a8a314c67df44845a6b558d969e92d64 is 50, key is test_row_0/A:col10/1732536330991/Put/seqid=0 2024-11-25T12:05:31,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742263_1439 (size=12001) 2024-11-25T12:05:31,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/a8a314c67df44845a6b558d969e92d64 2024-11-25T12:05:31,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/da981b2b29b541ffabebd807ae246887 is 50, key is test_row_0/B:col10/1732536330991/Put/seqid=0 2024-11-25T12:05:31,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T12:05:31,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742264_1440 (size=12001) 2024-11-25T12:05:31,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/da981b2b29b541ffabebd807ae246887 2024-11-25T12:05:31,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536391069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536391070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536391088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536391089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536391092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/a0fe8f7577b34d33b75552a509b1b141 is 50, key is test_row_0/C:col10/1732536330991/Put/seqid=0 2024-11-25T12:05:31,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:05:31,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:31,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:31,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:31,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:31,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:31,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:31,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742265_1441 (size=12001) 2024-11-25T12:05:31,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536391196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536391196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536391216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536391216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536391216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T12:05:31,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:05:31,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:31,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:31,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:31,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:31,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:31,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:31,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536391416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536391417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536391436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536391436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536391436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:05:31,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:31,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:31,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:31,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:31,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:31,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:31,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/a0fe8f7577b34d33b75552a509b1b141 2024-11-25T12:05:31,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/a8a314c67df44845a6b558d969e92d64 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/a8a314c67df44845a6b558d969e92d64 2024-11-25T12:05:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T12:05:31,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/a8a314c67df44845a6b558d969e92d64, entries=150, sequenceid=14, filesize=11.7 K 2024-11-25T12:05:31,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/da981b2b29b541ffabebd807ae246887 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/da981b2b29b541ffabebd807ae246887 2024-11-25T12:05:31,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/da981b2b29b541ffabebd807ae246887, entries=150, sequenceid=14, filesize=11.7 K 2024-11-25T12:05:31,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/a0fe8f7577b34d33b75552a509b1b141 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/a0fe8f7577b34d33b75552a509b1b141 2024-11-25T12:05:31,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:05:31,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:31,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:31,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:31,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:31,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:31,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/a0fe8f7577b34d33b75552a509b1b141, entries=150, sequenceid=14, filesize=11.7 K 2024-11-25T12:05:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:31,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d2665826b05855fc4d139a82044b59e6 in 652ms, sequenceid=14, compaction requested=false 2024-11-25T12:05:31,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:31,734 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:31,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T12:05:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:31,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/35964a53b2f24da2a69aa04ff73f755d is 50, key is test_row_0/A:col10/1732536331733/Put/seqid=0 2024-11-25T12:05:31,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742266_1442 (size=14341) 2024-11-25T12:05:31,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536391767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536391769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536391772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536391773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536391766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:05:31,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:31,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:31,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:31,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:31,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:31,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:31,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536391896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536391896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536391896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536391896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:31,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536391901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:31,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:05:31,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:31,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:31,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:31,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:31,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:31,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:32,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:05:32,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:32,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:32,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:32,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:32,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:32,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T12:05:32,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:32,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536392116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:32,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:32,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:32,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536392119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:32,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:32,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:32,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536392124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:32,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536392124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:32,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536392132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:32,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/35964a53b2f24da2a69aa04ff73f755d 2024-11-25T12:05:32,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/9ae92ac112f14de6a896eb93576e7439 is 50, key is test_row_0/B:col10/1732536331733/Put/seqid=0 2024-11-25T12:05:32,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742267_1443 (size=12001) 2024-11-25T12:05:32,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:05:32,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:32,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:32,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:32,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:32,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:32,417 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T12:05:32,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:32,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536392430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:32,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:32,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536392437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:32,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:32,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:32,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:05:32,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:32,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:32,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:32,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:32,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:32,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:32,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536392446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:32,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536392446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:32,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:32,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:32,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536392446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:32,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:32,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:32,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:05:32,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:32,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:32,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:32,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:32,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:32,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:32,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/9ae92ac112f14de6a896eb93576e7439 2024-11-25T12:05:32,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/7baad29150d1417cb2e6c97f293579a3 is 50, key is test_row_0/C:col10/1732536331733/Put/seqid=0 2024-11-25T12:05:32,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742268_1444 (size=12001) 2024-11-25T12:05:32,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:05:32,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:32,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:32,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:32,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:32,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:32,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:05:32,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:32,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:32,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:32,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:32,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:32,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536392953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:32,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:32,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536392953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:32,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:32,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536392966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:32,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536392966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:32,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:32,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:32,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536392971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:33,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/7baad29150d1417cb2e6c97f293579a3 2024-11-25T12:05:33,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:05:33,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:33,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:33,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:33,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:33,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:33,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:33,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T12:05:33,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/35964a53b2f24da2a69aa04ff73f755d as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/35964a53b2f24da2a69aa04ff73f755d 2024-11-25T12:05:33,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/35964a53b2f24da2a69aa04ff73f755d, entries=200, sequenceid=38, filesize=14.0 K 2024-11-25T12:05:33,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/9ae92ac112f14de6a896eb93576e7439 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/9ae92ac112f14de6a896eb93576e7439 2024-11-25T12:05:33,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/9ae92ac112f14de6a896eb93576e7439, entries=150, sequenceid=38, filesize=11.7 K 2024-11-25T12:05:33,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/7baad29150d1417cb2e6c97f293579a3 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/7baad29150d1417cb2e6c97f293579a3 2024-11-25T12:05:33,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/7baad29150d1417cb2e6c97f293579a3, entries=150, sequenceid=38, filesize=11.7 K 2024-11-25T12:05:33,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d2665826b05855fc4d139a82044b59e6 in 1535ms, sequenceid=38, compaction requested=false 2024-11-25T12:05:33,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:33,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-25T12:05:33,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:33,277 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T12:05:33,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/63eb68b69ba24e869a2d603d2d0a98c0 is 50, key is test_row_0/A:col10/1732536331770/Put/seqid=0 2024-11-25T12:05:33,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742269_1445 (size=12001) 2024-11-25T12:05:33,740 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/63eb68b69ba24e869a2d603d2d0a98c0 2024-11-25T12:05:33,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/4a187489782847b6a723baea6bd16697 is 50, key is test_row_0/B:col10/1732536331770/Put/seqid=0 2024-11-25T12:05:33,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742270_1446 (size=12001) 2024-11-25T12:05:33,807 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/4a187489782847b6a723baea6bd16697 2024-11-25T12:05:33,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/ba0b00e05b904cd58dc892e34abeb002 is 50, key is test_row_0/C:col10/1732536331770/Put/seqid=0 2024-11-25T12:05:33,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742271_1447 (size=12001) 2024-11-25T12:05:33,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:34,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536394048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536394061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536394062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536394064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536394066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536394176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536394187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536394187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536394196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536394196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,284 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/ba0b00e05b904cd58dc892e34abeb002 2024-11-25T12:05:34,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/63eb68b69ba24e869a2d603d2d0a98c0 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/63eb68b69ba24e869a2d603d2d0a98c0 2024-11-25T12:05:34,316 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/63eb68b69ba24e869a2d603d2d0a98c0, entries=150, sequenceid=50, filesize=11.7 K 2024-11-25T12:05:34,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-25T12:05:34,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/4a187489782847b6a723baea6bd16697 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/4a187489782847b6a723baea6bd16697 2024-11-25T12:05:34,328 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/4a187489782847b6a723baea6bd16697, entries=150, sequenceid=50, filesize=11.7 K 2024-11-25T12:05:34,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/ba0b00e05b904cd58dc892e34abeb002 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/ba0b00e05b904cd58dc892e34abeb002 2024-11-25T12:05:34,336 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/ba0b00e05b904cd58dc892e34abeb002, entries=150, sequenceid=50, filesize=11.7 K 2024-11-25T12:05:34,341 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for d2665826b05855fc4d139a82044b59e6 in 1064ms, sequenceid=50, compaction requested=true 2024-11-25T12:05:34,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:34,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:34,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-25T12:05:34,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-25T12:05:34,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-25T12:05:34,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3700 sec 2024-11-25T12:05:34,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 3.4130 sec 2024-11-25T12:05:34,400 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:34,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T12:05:34,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:34,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/ef481c654cd24409adb5bf74da56790d is 50, key is test_row_0/A:col10/1732536334064/Put/seqid=0 2024-11-25T12:05:34,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742272_1448 (size=12001) 2024-11-25T12:05:34,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/ef481c654cd24409adb5bf74da56790d 2024-11-25T12:05:34,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/da825122a5ba42cc8be16547869685d8 is 50, key is test_row_0/B:col10/1732536334064/Put/seqid=0 2024-11-25T12:05:34,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536394433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536394435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536394441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742273_1449 (size=12001) 2024-11-25T12:05:34,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/da825122a5ba42cc8be16547869685d8 2024-11-25T12:05:34,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536394461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536394465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/fea0bc1935584ec08c7b5ad54452e57c is 50, key is test_row_0/C:col10/1732536334064/Put/seqid=0 2024-11-25T12:05:34,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742274_1450 (size=12001) 2024-11-25T12:05:34,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536394569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536394569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536394580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536394586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536394599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,685 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T12:05:34,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536394799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536394799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536394821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536394825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536394826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:34,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:34,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:34,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/fea0bc1935584ec08c7b5ad54452e57c 2024-11-25T12:05:34,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/ef481c654cd24409adb5bf74da56790d as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/ef481c654cd24409adb5bf74da56790d 2024-11-25T12:05:35,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/ef481c654cd24409adb5bf74da56790d, entries=150, sequenceid=75, filesize=11.7 K 2024-11-25T12:05:35,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/da825122a5ba42cc8be16547869685d8 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/da825122a5ba42cc8be16547869685d8 2024-11-25T12:05:35,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/da825122a5ba42cc8be16547869685d8, entries=150, sequenceid=75, filesize=11.7 K 2024-11-25T12:05:35,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/fea0bc1935584ec08c7b5ad54452e57c as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/fea0bc1935584ec08c7b5ad54452e57c 2024-11-25T12:05:35,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:35,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536395117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:35,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/fea0bc1935584ec08c7b5ad54452e57c, entries=150, sequenceid=75, filesize=11.7 K 2024-11-25T12:05:35,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d2665826b05855fc4d139a82044b59e6 in 735ms, sequenceid=75, compaction requested=true 2024-11-25T12:05:35,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:35,136 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:35,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:05:35,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:35,136 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:35,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:05:35,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:35,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:35,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:35,142 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:35,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T12:05:35,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:35,149 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:35,150 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): d2665826b05855fc4d139a82044b59e6/A is initiating minor compaction (all files) 2024-11-25T12:05:35,150 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2665826b05855fc4d139a82044b59e6/A in TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:35,150 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/a8a314c67df44845a6b558d969e92d64, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/35964a53b2f24da2a69aa04ff73f755d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/63eb68b69ba24e869a2d603d2d0a98c0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/ef481c654cd24409adb5bf74da56790d] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp, totalSize=49.2 K 2024-11-25T12:05:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-25T12:05:35,158 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:35,159 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:05:35,163 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:35,163 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8a314c67df44845a6b558d969e92d64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732536330983 2024-11-25T12:05:35,163 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): d2665826b05855fc4d139a82044b59e6/B is initiating minor compaction (all files) 2024-11-25T12:05:35,163 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2665826b05855fc4d139a82044b59e6/B in TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:35,163 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/da981b2b29b541ffabebd807ae246887, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/9ae92ac112f14de6a896eb93576e7439, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/4a187489782847b6a723baea6bd16697, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/da825122a5ba42cc8be16547869685d8] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp, totalSize=46.9 K 2024-11-25T12:05:35,164 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35964a53b2f24da2a69aa04ff73f755d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732536331041 2024-11-25T12:05:35,165 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63eb68b69ba24e869a2d603d2d0a98c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732536331762 2024-11-25T12:05:35,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-25T12:05:35,165 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting da981b2b29b541ffabebd807ae246887, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732536330983 2024-11-25T12:05:35,166 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ae92ac112f14de6a896eb93576e7439, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732536331087 2024-11-25T12:05:35,166 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef481c654cd24409adb5bf74da56790d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732536334062 2024-11-25T12:05:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T12:05:35,167 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a187489782847b6a723baea6bd16697, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732536331762 2024-11-25T12:05:35,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/d9598553862341b28685c8cdbcd67a6d is 50, key is test_row_0/A:col10/1732536335133/Put/seqid=0 2024-11-25T12:05:35,168 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:05:35,168 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting da825122a5ba42cc8be16547869685d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732536334062 2024-11-25T12:05:35,173 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:05:35,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:05:35,187 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2665826b05855fc4d139a82044b59e6#A#compaction#373 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:35,188 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/4c3713cdf5e3446bad6b88b8034be565 is 50, key is test_row_0/A:col10/1732536334064/Put/seqid=0 2024-11-25T12:05:35,191 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2665826b05855fc4d139a82044b59e6#B#compaction#374 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:35,192 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/65ae3a0265d54be3802e5a3de9ffae96 is 50, key is test_row_0/B:col10/1732536334064/Put/seqid=0 2024-11-25T12:05:35,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742275_1451 (size=14341) 2024-11-25T12:05:35,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/d9598553862341b28685c8cdbcd67a6d 2024-11-25T12:05:35,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742277_1453 (size=12139) 2024-11-25T12:05:35,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742276_1452 (size=12139) 2024-11-25T12:05:35,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/c1b212d932884cf98ee2155f72aeb60a is 50, key is test_row_0/B:col10/1732536335133/Put/seqid=0 2024-11-25T12:05:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T12:05:35,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742278_1454 (size=12001) 2024-11-25T12:05:35,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T12:05:35,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:35,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536395326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:35,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536395328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:35,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:35,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536395333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:35,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:35,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:35,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:35,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:35,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:35,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:35,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536395349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:35,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:35,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536395456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:35,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536395456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:35,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:35,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:35,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536395456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:35,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:35,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T12:05:35,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:35,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536395476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:35,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T12:05:35,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:35,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:35,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:35,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:35,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:35,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:35,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536395629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:35,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:35,663 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/65ae3a0265d54be3802e5a3de9ffae96 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/65ae3a0265d54be3802e5a3de9ffae96 2024-11-25T12:05:35,673 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/4c3713cdf5e3446bad6b88b8034be565 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/4c3713cdf5e3446bad6b88b8034be565 2024-11-25T12:05:35,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T12:05:35,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:35,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:35,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:35,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:35,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:35,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:35,681 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2665826b05855fc4d139a82044b59e6/B of d2665826b05855fc4d139a82044b59e6 into 65ae3a0265d54be3802e5a3de9ffae96(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:35,681 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:35,681 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., storeName=d2665826b05855fc4d139a82044b59e6/B, priority=12, startTime=1732536335136; duration=0sec 2024-11-25T12:05:35,681 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:35,681 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:B 2024-11-25T12:05:35,681 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:35,689 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2665826b05855fc4d139a82044b59e6/A of d2665826b05855fc4d139a82044b59e6 into 4c3713cdf5e3446bad6b88b8034be565(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:35,689 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:35,689 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., storeName=d2665826b05855fc4d139a82044b59e6/A, priority=12, startTime=1732536335135; duration=0sec 2024-11-25T12:05:35,689 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:35,689 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:A 2024-11-25T12:05:35,692 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:35,692 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): d2665826b05855fc4d139a82044b59e6/C is initiating minor compaction (all files) 2024-11-25T12:05:35,692 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2665826b05855fc4d139a82044b59e6/C in TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:35,692 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/a0fe8f7577b34d33b75552a509b1b141, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/7baad29150d1417cb2e6c97f293579a3, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/ba0b00e05b904cd58dc892e34abeb002, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/fea0bc1935584ec08c7b5ad54452e57c] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp, totalSize=46.9 K 2024-11-25T12:05:35,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536395676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:35,694 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting a0fe8f7577b34d33b75552a509b1b141, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732536330983 2024-11-25T12:05:35,694 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 7baad29150d1417cb2e6c97f293579a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732536331087 2024-11-25T12:05:35,695 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting ba0b00e05b904cd58dc892e34abeb002, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732536331762 2024-11-25T12:05:35,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:35,695 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting fea0bc1935584ec08c7b5ad54452e57c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732536334062 2024-11-25T12:05:35,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536395686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:35,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:35,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536395688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:35,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:35,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/c1b212d932884cf98ee2155f72aeb60a 2024-11-25T12:05:35,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536395696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:35,727 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2665826b05855fc4d139a82044b59e6#C#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:35,727 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/4e54597d5b4d45c88e2a8790607e6988 is 50, key is test_row_0/C:col10/1732536334064/Put/seqid=0 2024-11-25T12:05:35,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/3f384a9b16724546bbc9009c73f56859 is 50, key is test_row_0/C:col10/1732536335133/Put/seqid=0 2024-11-25T12:05:35,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742279_1455 (size=12139) 2024-11-25T12:05:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T12:05:35,807 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/4e54597d5b4d45c88e2a8790607e6988 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/4e54597d5b4d45c88e2a8790607e6988 2024-11-25T12:05:35,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742280_1456 (size=12001) 2024-11-25T12:05:35,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T12:05:35,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:35,829 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2665826b05855fc4d139a82044b59e6/C of d2665826b05855fc4d139a82044b59e6 into 4e54597d5b4d45c88e2a8790607e6988(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:35,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:35,829 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:35,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:35,830 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., storeName=d2665826b05855fc4d139a82044b59e6/C, priority=12, startTime=1732536335136; duration=0sec 2024-11-25T12:05:35,830 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:35,830 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:C 2024-11-25T12:05:35,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:35,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:36,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T12:05:36,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:36,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:36,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:36,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:36,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:36,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:36,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536396010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:36,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:36,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536396010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:36,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:36,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536396012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:36,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:36,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:36,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:36,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:36,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536396028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:36,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:36,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T12:05:36,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:36,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:36,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:36,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:36,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:36,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/3f384a9b16724546bbc9009c73f56859 2024-11-25T12:05:36,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/d9598553862341b28685c8cdbcd67a6d as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/d9598553862341b28685c8cdbcd67a6d 2024-11-25T12:05:36,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/d9598553862341b28685c8cdbcd67a6d, entries=200, sequenceid=87, filesize=14.0 K 2024-11-25T12:05:36,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/c1b212d932884cf98ee2155f72aeb60a as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/c1b212d932884cf98ee2155f72aeb60a 2024-11-25T12:05:36,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/c1b212d932884cf98ee2155f72aeb60a, entries=150, sequenceid=87, filesize=11.7 K 2024-11-25T12:05:36,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/3f384a9b16724546bbc9009c73f56859 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/3f384a9b16724546bbc9009c73f56859 2024-11-25T12:05:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T12:05:36,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/3f384a9b16724546bbc9009c73f56859, entries=150, sequenceid=87, filesize=11.7 K 2024-11-25T12:05:36,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T12:05:36,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:36,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:36,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:36,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:36,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:36,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for d2665826b05855fc4d139a82044b59e6 in 1202ms, sequenceid=87, compaction requested=false 2024-11-25T12:05:36,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:36,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-25T12:05:36,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:36,494 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T12:05:36,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/54591a2814ef4246a82e0f9f2917b647 is 50, key is test_row_0/A:col10/1732536335325/Put/seqid=0 2024-11-25T12:05:36,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742281_1457 (size=12001) 2024-11-25T12:05:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:36,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:36,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536396589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:36,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,615 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:36,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536396601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:36,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536396613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:36,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536396613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:36,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536396649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:36,661 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_1' in d2665826b05855fc4d139a82044b59e6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:05:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:36,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536396727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:36,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:36,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536396732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:36,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:36,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536396746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536396747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:36,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:36,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:36,945 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/54591a2814ef4246a82e0f9f2917b647 2024-11-25T12:05:36,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:36,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536396948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:36,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:36,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:36,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536396956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:36,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:36,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536396969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:36,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:36,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536396980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:36,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/7282706a130548a48cb354431b9c4b88 is 50, key is test_row_0/B:col10/1732536335325/Put/seqid=0 2024-11-25T12:05:37,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742282_1458 (size=12001) 2024-11-25T12:05:37,060 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/7282706a130548a48cb354431b9c4b88 2024-11-25T12:05:37,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/b40c6f55603a43dfa079b9eb92149a9e is 50, key is test_row_0/C:col10/1732536335325/Put/seqid=0 2024-11-25T12:05:37,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742283_1459 (size=12001) 2024-11-25T12:05:37,200 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/b40c6f55603a43dfa079b9eb92149a9e 2024-11-25T12:05:37,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/54591a2814ef4246a82e0f9f2917b647 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/54591a2814ef4246a82e0f9f2917b647 2024-11-25T12:05:37,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:37,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536397266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:37,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:37,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:37,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:37,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536397277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:37,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:37,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:37,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:37,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536397286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:37,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:37,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:37,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536397298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:37,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:37,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:37,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:37,329 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/54591a2814ef4246a82e0f9f2917b647, entries=150, sequenceid=114, filesize=11.7 K 2024-11-25T12:05:37,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/7282706a130548a48cb354431b9c4b88 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/7282706a130548a48cb354431b9c4b88 2024-11-25T12:05:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T12:05:37,361 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/7282706a130548a48cb354431b9c4b88, entries=150, sequenceid=114, filesize=11.7 K 2024-11-25T12:05:37,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/b40c6f55603a43dfa079b9eb92149a9e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/b40c6f55603a43dfa079b9eb92149a9e 2024-11-25T12:05:37,397 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/b40c6f55603a43dfa079b9eb92149a9e, entries=150, sequenceid=114, filesize=11.7 K 2024-11-25T12:05:37,412 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for d2665826b05855fc4d139a82044b59e6 in 918ms, sequenceid=114, compaction requested=true 2024-11-25T12:05:37,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:37,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:37,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-25T12:05:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-25T12:05:37,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-25T12:05:37,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2430 sec 2024-11-25T12:05:37,437 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.2690 sec 2024-11-25T12:05:37,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:37,816 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:37,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-25T12:05:37,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/57d3ab8d918847f8b347eadfffdb6305 is 50, key is test_row_0/A:col10/1732536337808/Put/seqid=0 2024-11-25T12:05:37,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742284_1460 (size=12051) 2024-11-25T12:05:37,893 WARN [regionserver/2bf237fd86ca:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-25T12:05:37,897 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2bf237fd86ca:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/a8a314c67df44845a6b558d969e92d64, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/35964a53b2f24da2a69aa04ff73f755d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/63eb68b69ba24e869a2d603d2d0a98c0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/ef481c654cd24409adb5bf74da56790d] to archive 2024-11-25T12:05:37,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/57d3ab8d918847f8b347eadfffdb6305 2024-11-25T12:05:37,904 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2bf237fd86ca:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:05:37,916 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2bf237fd86ca:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/a8a314c67df44845a6b558d969e92d64 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/a8a314c67df44845a6b558d969e92d64 2024-11-25T12:05:37,928 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2bf237fd86ca:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/35964a53b2f24da2a69aa04ff73f755d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/35964a53b2f24da2a69aa04ff73f755d 2024-11-25T12:05:37,935 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2bf237fd86ca:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/63eb68b69ba24e869a2d603d2d0a98c0 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/63eb68b69ba24e869a2d603d2d0a98c0 2024-11-25T12:05:37,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/c14238af66604261ae9ef0443ce05730 is 50, key is test_row_0/B:col10/1732536337808/Put/seqid=0 2024-11-25T12:05:37,952 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2bf237fd86ca:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/ef481c654cd24409adb5bf74da56790d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/ef481c654cd24409adb5bf74da56790d 2024-11-25T12:05:37,957 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2bf237fd86ca:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/da981b2b29b541ffabebd807ae246887, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/9ae92ac112f14de6a896eb93576e7439, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/4a187489782847b6a723baea6bd16697, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/da825122a5ba42cc8be16547869685d8] to archive 2024-11-25T12:05:37,958 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2bf237fd86ca:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:05:37,969 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2bf237fd86ca:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/da981b2b29b541ffabebd807ae246887 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/da981b2b29b541ffabebd807ae246887 2024-11-25T12:05:37,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536397945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:37,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536397945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:37,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:37,975 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2bf237fd86ca:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/9ae92ac112f14de6a896eb93576e7439 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/9ae92ac112f14de6a896eb93576e7439 2024-11-25T12:05:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536397945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:37,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536397945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:37,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:37,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:37,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:37,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:37,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:37,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:37,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:37,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:37,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:37,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:37,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:37,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:37,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742285_1461 (size=12051) 2024-11-25T12:05:37,981 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2bf237fd86ca:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/4a187489782847b6a723baea6bd16697 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/4a187489782847b6a723baea6bd16697 2024-11-25T12:05:37,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/c14238af66604261ae9ef0443ce05730 2024-11-25T12:05:37,993 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2bf237fd86ca:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/da825122a5ba42cc8be16547869685d8 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/da825122a5ba42cc8be16547869685d8 2024-11-25T12:05:37,996 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2bf237fd86ca:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/a0fe8f7577b34d33b75552a509b1b141, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/7baad29150d1417cb2e6c97f293579a3, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/ba0b00e05b904cd58dc892e34abeb002, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/fea0bc1935584ec08c7b5ad54452e57c] to archive 2024-11-25T12:05:38,001 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2bf237fd86ca:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:05:38,029 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2bf237fd86ca:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/a0fe8f7577b34d33b75552a509b1b141 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/a0fe8f7577b34d33b75552a509b1b141 2024-11-25T12:05:38,049 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2bf237fd86ca:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/7baad29150d1417cb2e6c97f293579a3 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/7baad29150d1417cb2e6c97f293579a3 2024-11-25T12:05:38,077 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2bf237fd86ca:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/ba0b00e05b904cd58dc892e34abeb002 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/ba0b00e05b904cd58dc892e34abeb002 2024-11-25T12:05:38,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/413497266ed14706b477f75b5294de8f is 50, key is test_row_0/C:col10/1732536337808/Put/seqid=0 2024-11-25T12:05:38,088 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2bf237fd86ca:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/fea0bc1935584ec08c7b5ad54452e57c to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/fea0bc1935584ec08c7b5ad54452e57c 2024-11-25T12:05:38,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:38,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536398086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:38,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:38,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:38,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536398087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:38,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:38,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:38,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:38,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536398087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:38,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536398087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:38,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742286_1462 (size=12051) 2024-11-25T12:05:38,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/413497266ed14706b477f75b5294de8f 2024-11-25T12:05:38,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/57d3ab8d918847f8b347eadfffdb6305 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/57d3ab8d918847f8b347eadfffdb6305 2024-11-25T12:05:38,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/57d3ab8d918847f8b347eadfffdb6305, entries=150, sequenceid=129, filesize=11.8 K 2024-11-25T12:05:38,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/c14238af66604261ae9ef0443ce05730 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/c14238af66604261ae9ef0443ce05730 2024-11-25T12:05:38,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/c14238af66604261ae9ef0443ce05730, entries=150, sequenceid=129, filesize=11.8 K 2024-11-25T12:05:38,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/413497266ed14706b477f75b5294de8f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/413497266ed14706b477f75b5294de8f 2024-11-25T12:05:38,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536398317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:38,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:38,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536398317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:38,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536398317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:38,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:38,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536398321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:38,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:38,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/413497266ed14706b477f75b5294de8f, entries=150, sequenceid=129, filesize=11.8 K 2024-11-25T12:05:38,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d2665826b05855fc4d139a82044b59e6 in 515ms, sequenceid=129, compaction requested=true 2024-11-25T12:05:38,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:38,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:05:38,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:38,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:05:38,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:38,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:38,332 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:38,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:05:38,332 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:38,339 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:38,339 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): d2665826b05855fc4d139a82044b59e6/B is initiating minor compaction (all files) 2024-11-25T12:05:38,339 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2665826b05855fc4d139a82044b59e6/B in TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:38,339 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/65ae3a0265d54be3802e5a3de9ffae96, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/c1b212d932884cf98ee2155f72aeb60a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/7282706a130548a48cb354431b9c4b88, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/c14238af66604261ae9ef0443ce05730] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp, totalSize=47.1 K 2024-11-25T12:05:38,339 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:38,339 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): d2665826b05855fc4d139a82044b59e6/A is initiating minor compaction (all files) 2024-11-25T12:05:38,339 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2665826b05855fc4d139a82044b59e6/A in TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:38,340 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/4c3713cdf5e3446bad6b88b8034be565, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/d9598553862341b28685c8cdbcd67a6d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/54591a2814ef4246a82e0f9f2917b647, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/57d3ab8d918847f8b347eadfffdb6305] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp, totalSize=49.3 K 2024-11-25T12:05:38,340 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65ae3a0265d54be3802e5a3de9ffae96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732536334062 2024-11-25T12:05:38,340 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c3713cdf5e3446bad6b88b8034be565, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732536334062 2024-11-25T12:05:38,340 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1b212d932884cf98ee2155f72aeb60a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732536334421 2024-11-25T12:05:38,345 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7282706a130548a48cb354431b9c4b88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732536335324 2024-11-25T12:05:38,345 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting c14238af66604261ae9ef0443ce05730, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732536336587 2024-11-25T12:05:38,346 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting d9598553862341b28685c8cdbcd67a6d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732536334421 2024-11-25T12:05:38,346 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 54591a2814ef4246a82e0f9f2917b647, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732536335324 2024-11-25T12:05:38,346 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 57d3ab8d918847f8b347eadfffdb6305, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732536336587 2024-11-25T12:05:38,371 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2665826b05855fc4d139a82044b59e6#B#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:38,372 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/19a1c8ca8ce948cfaec4e1c7d043317a is 50, key is test_row_0/B:col10/1732536337808/Put/seqid=0 2024-11-25T12:05:38,378 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2665826b05855fc4d139a82044b59e6#A#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:38,378 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/580b092675824376ae286c5c7dea9d8f is 50, key is test_row_0/A:col10/1732536337808/Put/seqid=0 2024-11-25T12:05:38,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742287_1463 (size=12189) 2024-11-25T12:05:38,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742288_1464 (size=12189) 2024-11-25T12:05:38,451 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/580b092675824376ae286c5c7dea9d8f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/580b092675824376ae286c5c7dea9d8f 2024-11-25T12:05:38,457 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2665826b05855fc4d139a82044b59e6/A of d2665826b05855fc4d139a82044b59e6 into 580b092675824376ae286c5c7dea9d8f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:38,457 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:38,457 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., storeName=d2665826b05855fc4d139a82044b59e6/A, priority=12, startTime=1732536338332; duration=0sec 2024-11-25T12:05:38,457 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:38,457 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:A 2024-11-25T12:05:38,457 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:38,458 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:38,458 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): d2665826b05855fc4d139a82044b59e6/C is initiating minor compaction (all files) 2024-11-25T12:05:38,458 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2665826b05855fc4d139a82044b59e6/C in TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:38,458 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/4e54597d5b4d45c88e2a8790607e6988, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/3f384a9b16724546bbc9009c73f56859, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/b40c6f55603a43dfa079b9eb92149a9e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/413497266ed14706b477f75b5294de8f] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp, totalSize=47.1 K 2024-11-25T12:05:38,459 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e54597d5b4d45c88e2a8790607e6988, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732536334062 2024-11-25T12:05:38,459 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f384a9b16724546bbc9009c73f56859, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732536334421 2024-11-25T12:05:38,459 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting b40c6f55603a43dfa079b9eb92149a9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732536335324 2024-11-25T12:05:38,460 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 413497266ed14706b477f75b5294de8f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732536336587 2024-11-25T12:05:38,481 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2665826b05855fc4d139a82044b59e6#C#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:38,482 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/23c3c5884c0041aab9e4710d109621eb is 50, key is test_row_0/C:col10/1732536337808/Put/seqid=0 2024-11-25T12:05:38,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742289_1465 (size=12189) 2024-11-25T12:05:38,645 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:38,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T12:05:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:38,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/497f499bd9494631ae2f692cc4f1a212 is 50, key is test_row_0/A:col10/1732536337910/Put/seqid=0 2024-11-25T12:05:38,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742290_1466 (size=14541) 2024-11-25T12:05:38,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536398697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:38,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:38,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536398706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:38,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536398707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:38,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:38,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:38,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536398713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:38,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536398716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:38,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536398828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:38,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:38,853 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/19a1c8ca8ce948cfaec4e1c7d043317a as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/19a1c8ca8ce948cfaec4e1c7d043317a 2024-11-25T12:05:38,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536398848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:38,861 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2665826b05855fc4d139a82044b59e6/B of d2665826b05855fc4d139a82044b59e6 into 19a1c8ca8ce948cfaec4e1c7d043317a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:38,861 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:38,861 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., storeName=d2665826b05855fc4d139a82044b59e6/B, priority=12, startTime=1732536338332; duration=0sec 2024-11-25T12:05:38,861 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:38,861 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:B 2024-11-25T12:05:38,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:38,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536398848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:38,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536398850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:38,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536398856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:38,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:38,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:38,937 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/23c3c5884c0041aab9e4710d109621eb as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/23c3c5884c0041aab9e4710d109621eb 2024-11-25T12:05:38,945 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2665826b05855fc4d139a82044b59e6/C of d2665826b05855fc4d139a82044b59e6 into 23c3c5884c0041aab9e4710d109621eb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:38,945 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:38,945 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., storeName=d2665826b05855fc4d139a82044b59e6/C, priority=12, startTime=1732536338332; duration=0sec 2024-11-25T12:05:38,946 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:38,946 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:C 2024-11-25T12:05:39,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:39,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536399057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:39,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536399066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:39,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:39,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:39,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536399066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:39,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536399068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:39,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536399069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:39,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:39,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:39,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/497f499bd9494631ae2f692cc4f1a212 2024-11-25T12:05:39,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/5ee9d451b4c94d459986222b3ccc5cce is 50, key is test_row_0/B:col10/1732536337910/Put/seqid=0 2024-11-25T12:05:39,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742291_1467 (size=12151) 2024-11-25T12:05:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-25T12:05:39,368 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:39,369 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:05:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-25T12:05:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T12:05:39,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536399376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536399376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:39,396 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:05:39,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536399387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:39,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:39,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536399387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:39,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:39,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536399387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:39,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:39,404 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:05:39,405 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:05:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T12:05:39,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-25T12:05:39,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:39,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:39,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:39,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:39,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:39,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:39,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/5ee9d451b4c94d459986222b3ccc5cce 2024-11-25T12:05:39,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/dede5c64a5f94a2bb9ad490d3b373786 is 50, key is test_row_0/C:col10/1732536337910/Put/seqid=0 2024-11-25T12:05:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T12:05:39,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-25T12:05:39,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:39,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:39,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:39,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:39,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:39,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742292_1468 (size=12151) 2024-11-25T12:05:39,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-25T12:05:39,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:39,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:39,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:39,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:39,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:39,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536399896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536399896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:39,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536399906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:39,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536399907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:39,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536399907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:39,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T12:05:40,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-25T12:05:40,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:40,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:40,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:40,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:40,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:40,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/dede5c64a5f94a2bb9ad490d3b373786 2024-11-25T12:05:40,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/497f499bd9494631ae2f692cc4f1a212 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/497f499bd9494631ae2f692cc4f1a212 2024-11-25T12:05:40,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/497f499bd9494631ae2f692cc4f1a212, entries=200, sequenceid=152, filesize=14.2 K 2024-11-25T12:05:40,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/5ee9d451b4c94d459986222b3ccc5cce as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/5ee9d451b4c94d459986222b3ccc5cce 2024-11-25T12:05:40,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/5ee9d451b4c94d459986222b3ccc5cce, entries=150, sequenceid=152, filesize=11.9 K 2024-11-25T12:05:40,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/dede5c64a5f94a2bb9ad490d3b373786 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/dede5c64a5f94a2bb9ad490d3b373786 2024-11-25T12:05:40,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-25T12:05:40,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:40,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:40,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:40,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:40,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:40,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/dede5c64a5f94a2bb9ad490d3b373786, entries=150, sequenceid=152, filesize=11.9 K 2024-11-25T12:05:40,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:40,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for d2665826b05855fc4d139a82044b59e6 in 1604ms, sequenceid=152, compaction requested=false 2024-11-25T12:05:40,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:40,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-25T12:05:40,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:40,406 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-25T12:05:40,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/203d262762da46a9b661a1b894d77f57 is 50, key is test_row_0/A:col10/1732536338711/Put/seqid=0 2024-11-25T12:05:40,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742293_1469 (size=12151) 2024-11-25T12:05:40,507 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/203d262762da46a9b661a1b894d77f57 2024-11-25T12:05:40,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T12:05:40,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/423f7a8cd9f641d8b7514066c44a7b97 is 50, key is test_row_0/B:col10/1732536338711/Put/seqid=0 2024-11-25T12:05:40,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742294_1470 (size=12151) 2024-11-25T12:05:40,664 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/423f7a8cd9f641d8b7514066c44a7b97 2024-11-25T12:05:40,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/d12d148255bf4a4a97d8032508d5420f is 50, key is test_row_0/C:col10/1732536338711/Put/seqid=0 2024-11-25T12:05:40,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742295_1471 (size=12151) 2024-11-25T12:05:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:40,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:41,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536401005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536401013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536401025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536401025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536401026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536401136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536401139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536401157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536401159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536401159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,202 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/d12d148255bf4a4a97d8032508d5420f 2024-11-25T12:05:41,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/203d262762da46a9b661a1b894d77f57 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/203d262762da46a9b661a1b894d77f57 2024-11-25T12:05:41,253 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/203d262762da46a9b661a1b894d77f57, entries=150, sequenceid=169, filesize=11.9 K 2024-11-25T12:05:41,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/423f7a8cd9f641d8b7514066c44a7b97 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/423f7a8cd9f641d8b7514066c44a7b97 2024-11-25T12:05:41,258 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/423f7a8cd9f641d8b7514066c44a7b97, entries=150, sequenceid=169, filesize=11.9 K 2024-11-25T12:05:41,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/d12d148255bf4a4a97d8032508d5420f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/d12d148255bf4a4a97d8032508d5420f 2024-11-25T12:05:41,264 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/d12d148255bf4a4a97d8032508d5420f, entries=150, sequenceid=169, filesize=11.9 K 2024-11-25T12:05:41,265 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d2665826b05855fc4d139a82044b59e6 in 860ms, sequenceid=169, compaction requested=true 2024-11-25T12:05:41,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:41,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:41,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-25T12:05:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-25T12:05:41,272 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-25T12:05:41,272 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8660 sec 2024-11-25T12:05:41,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.9040 sec 2024-11-25T12:05:41,364 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:41,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T12:05:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:41,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/170c1a43ab264053a4f62d001c1f768b is 50, key is test_row_0/A:col10/1732536341022/Put/seqid=0 2024-11-25T12:05:41,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742296_1472 (size=14541) 2024-11-25T12:05:41,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536401421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536401484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536401484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536401485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536401488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-25T12:05:41,529 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:41,530 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:05:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-25T12:05:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T12:05:41,532 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:05:41,535 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:05:41,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:05:41,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536401597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536401617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536401617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536401617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536401621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T12:05:41,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T12:05:41,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:41,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:41,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:41,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:41,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:41,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:41,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/170c1a43ab264053a4f62d001c1f768b 2024-11-25T12:05:41,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536401816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536401837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536401837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536401841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T12:05:41,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:41,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:41,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:41,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:41,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T12:05:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536401846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:41,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/2e1d1ba51b754aa290c4e430796ff0f3 is 50, key is test_row_0/B:col10/1732536341022/Put/seqid=0 2024-11-25T12:05:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:41,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742297_1473 (size=12151) 2024-11-25T12:05:42,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T12:05:42,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:42,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:42,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:42,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:42,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:42,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:42,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536402138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T12:05:42,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:42,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536402157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:42,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T12:05:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536402156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:42,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536402160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:42,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:42,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:42,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:42,173 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536402161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:42,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:42,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:42,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:42,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:42,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:42,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:42,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:42,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/2e1d1ba51b754aa290c4e430796ff0f3 2024-11-25T12:05:42,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/4c316c9b3bd7408db0b53bfaaad155f2 is 50, key is test_row_0/C:col10/1732536341022/Put/seqid=0 2024-11-25T12:05:42,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T12:05:42,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:42,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:42,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:42,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:42,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:42,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742298_1474 (size=12151) 2024-11-25T12:05:42,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T12:05:42,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:42,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:42,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:42,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:42,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:42,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:42,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536402656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:42,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T12:05:42,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:42,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:42,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T12:05:42,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:42,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:42,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:42,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:42,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:42,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536402682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:42,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536402688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:42,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536402688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:42,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:42,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536402690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:42,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:42,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:42,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:42,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:42,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:42,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:42,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/4c316c9b3bd7408db0b53bfaaad155f2 2024-11-25T12:05:42,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/170c1a43ab264053a4f62d001c1f768b as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/170c1a43ab264053a4f62d001c1f768b 2024-11-25T12:05:42,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T12:05:42,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/170c1a43ab264053a4f62d001c1f768b, entries=200, sequenceid=191, filesize=14.2 K 2024-11-25T12:05:42,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:42,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:42,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:42,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:42,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:42,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/2e1d1ba51b754aa290c4e430796ff0f3 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/2e1d1ba51b754aa290c4e430796ff0f3 2024-11-25T12:05:42,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/2e1d1ba51b754aa290c4e430796ff0f3, entries=150, sequenceid=191, filesize=11.9 K 2024-11-25T12:05:42,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/4c316c9b3bd7408db0b53bfaaad155f2 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/4c316c9b3bd7408db0b53bfaaad155f2 2024-11-25T12:05:42,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/4c316c9b3bd7408db0b53bfaaad155f2, entries=150, sequenceid=191, filesize=11.9 K 2024-11-25T12:05:42,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for d2665826b05855fc4d139a82044b59e6 in 1508ms, sequenceid=191, compaction requested=true 2024-11-25T12:05:42,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:42,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:05:42,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:42,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:05:42,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:05:42,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:42,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:05:42,873 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:42,873 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:42,877 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53422 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:42,877 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): d2665826b05855fc4d139a82044b59e6/A is initiating minor compaction (all files) 2024-11-25T12:05:42,877 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2665826b05855fc4d139a82044b59e6/A in TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:42,877 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/580b092675824376ae286c5c7dea9d8f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/497f499bd9494631ae2f692cc4f1a212, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/203d262762da46a9b661a1b894d77f57, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/170c1a43ab264053a4f62d001c1f768b] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp, totalSize=52.2 K 2024-11-25T12:05:42,877 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48642 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:42,878 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): d2665826b05855fc4d139a82044b59e6/C is initiating minor compaction (all files) 2024-11-25T12:05:42,878 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2665826b05855fc4d139a82044b59e6/C in TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:42,878 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/23c3c5884c0041aab9e4710d109621eb, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/dede5c64a5f94a2bb9ad490d3b373786, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/d12d148255bf4a4a97d8032508d5420f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/4c316c9b3bd7408db0b53bfaaad155f2] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp, totalSize=47.5 K 2024-11-25T12:05:42,881 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 580b092675824376ae286c5c7dea9d8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732536336587 2024-11-25T12:05:42,881 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 23c3c5884c0041aab9e4710d109621eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732536336587 2024-11-25T12:05:42,884 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 497f499bd9494631ae2f692cc4f1a212, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732536337910 2024-11-25T12:05:42,885 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting dede5c64a5f94a2bb9ad490d3b373786, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732536337910 2024-11-25T12:05:42,888 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 203d262762da46a9b661a1b894d77f57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732536338695 2024-11-25T12:05:42,889 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting d12d148255bf4a4a97d8032508d5420f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732536338695 2024-11-25T12:05:42,892 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 170c1a43ab264053a4f62d001c1f768b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732536340999 2024-11-25T12:05:42,892 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c316c9b3bd7408db0b53bfaaad155f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732536341021 2024-11-25T12:05:42,945 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2665826b05855fc4d139a82044b59e6#A#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:42,945 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/a69ef55aba904ea9b5e6b1ec89eb981e is 50, key is test_row_0/A:col10/1732536341022/Put/seqid=0 2024-11-25T12:05:42,953 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2665826b05855fc4d139a82044b59e6#C#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:42,954 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/45969a05986a464ba2244cf40ae1ad16 is 50, key is test_row_0/C:col10/1732536341022/Put/seqid=0 2024-11-25T12:05:42,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-25T12:05:42,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:42,995 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-25T12:05:43,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742300_1476 (size=12425) 2024-11-25T12:05:43,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/deebb2ee24324349b6a0403ae5edd85f is 50, key is test_row_0/A:col10/1732536341481/Put/seqid=0 2024-11-25T12:05:43,017 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/45969a05986a464ba2244cf40ae1ad16 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/45969a05986a464ba2244cf40ae1ad16 2024-11-25T12:05:43,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742299_1475 (size=12425) 2024-11-25T12:05:43,023 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2665826b05855fc4d139a82044b59e6/C of d2665826b05855fc4d139a82044b59e6 into 45969a05986a464ba2244cf40ae1ad16(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:43,023 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:43,023 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., storeName=d2665826b05855fc4d139a82044b59e6/C, priority=12, startTime=1732536342872; duration=0sec 2024-11-25T12:05:43,023 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:43,023 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:C 2024-11-25T12:05:43,023 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:43,042 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48642 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:43,043 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): d2665826b05855fc4d139a82044b59e6/B is initiating minor compaction (all files) 2024-11-25T12:05:43,043 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2665826b05855fc4d139a82044b59e6/B in TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:43,043 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/19a1c8ca8ce948cfaec4e1c7d043317a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/5ee9d451b4c94d459986222b3ccc5cce, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/423f7a8cd9f641d8b7514066c44a7b97, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/2e1d1ba51b754aa290c4e430796ff0f3] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp, totalSize=47.5 K 2024-11-25T12:05:43,045 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 19a1c8ca8ce948cfaec4e1c7d043317a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732536336587 2024-11-25T12:05:43,049 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ee9d451b4c94d459986222b3ccc5cce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732536337910 2024-11-25T12:05:43,053 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 423f7a8cd9f641d8b7514066c44a7b97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732536338695 2024-11-25T12:05:43,057 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e1d1ba51b754aa290c4e430796ff0f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732536341021 2024-11-25T12:05:43,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742301_1477 (size=12151) 2024-11-25T12:05:43,069 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/deebb2ee24324349b6a0403ae5edd85f 2024-11-25T12:05:43,081 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/a69ef55aba904ea9b5e6b1ec89eb981e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/a69ef55aba904ea9b5e6b1ec89eb981e 2024-11-25T12:05:43,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/176f121b6a5c467e93356b672ce48be7 is 50, key is test_row_0/B:col10/1732536341481/Put/seqid=0 2024-11-25T12:05:43,122 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2665826b05855fc4d139a82044b59e6/A of d2665826b05855fc4d139a82044b59e6 into a69ef55aba904ea9b5e6b1ec89eb981e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:43,122 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:43,122 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., storeName=d2665826b05855fc4d139a82044b59e6/A, priority=12, startTime=1732536342872; duration=0sec 2024-11-25T12:05:43,122 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:43,122 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:A 2024-11-25T12:05:43,127 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2665826b05855fc4d139a82044b59e6#B#compaction#400 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:43,127 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/f8ab7a45d7d44dc296e071e6070f851e is 50, key is test_row_0/B:col10/1732536341022/Put/seqid=0 2024-11-25T12:05:43,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742303_1479 (size=12425) 2024-11-25T12:05:43,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742302_1478 (size=12151) 2024-11-25T12:05:43,165 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/176f121b6a5c467e93356b672ce48be7 2024-11-25T12:05:43,185 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/f8ab7a45d7d44dc296e071e6070f851e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/f8ab7a45d7d44dc296e071e6070f851e 2024-11-25T12:05:43,193 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2665826b05855fc4d139a82044b59e6/B of d2665826b05855fc4d139a82044b59e6 into f8ab7a45d7d44dc296e071e6070f851e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:43,193 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:43,193 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., storeName=d2665826b05855fc4d139a82044b59e6/B, priority=12, startTime=1732536342872; duration=0sec 2024-11-25T12:05:43,193 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:43,193 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:B 2024-11-25T12:05:43,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/c49c9a1b40f94bcf823ead26934f1cd0 is 50, key is test_row_0/C:col10/1732536341481/Put/seqid=0 2024-11-25T12:05:43,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742304_1480 (size=12151) 2024-11-25T12:05:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T12:05:43,684 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/c49c9a1b40f94bcf823ead26934f1cd0 2024-11-25T12:05:43,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:43,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:43,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/deebb2ee24324349b6a0403ae5edd85f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/deebb2ee24324349b6a0403ae5edd85f 2024-11-25T12:05:43,701 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/deebb2ee24324349b6a0403ae5edd85f, entries=150, sequenceid=206, filesize=11.9 K 2024-11-25T12:05:43,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/176f121b6a5c467e93356b672ce48be7 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/176f121b6a5c467e93356b672ce48be7 2024-11-25T12:05:43,726 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/176f121b6a5c467e93356b672ce48be7, entries=150, sequenceid=206, filesize=11.9 K 2024-11-25T12:05:43,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/c49c9a1b40f94bcf823ead26934f1cd0 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/c49c9a1b40f94bcf823ead26934f1cd0 2024-11-25T12:05:43,745 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/c49c9a1b40f94bcf823ead26934f1cd0, entries=150, sequenceid=206, filesize=11.9 K 2024-11-25T12:05:43,746 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=87.22 KB/89310 for d2665826b05855fc4d139a82044b59e6 in 751ms, sequenceid=206, compaction requested=false 2024-11-25T12:05:43,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:43,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:43,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-25T12:05:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-25T12:05:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:43,748 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:43,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-25T12:05:43,752 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-25T12:05:43,752 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2130 sec 2024-11-25T12:05:43,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.2230 sec 2024-11-25T12:05:43,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/a327600ea4104b2f98a431b029627ae6 is 50, key is test_row_0/A:col10/1732536343730/Put/seqid=0 2024-11-25T12:05:43,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742305_1481 (size=14541) 2024-11-25T12:05:43,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/a327600ea4104b2f98a431b029627ae6 2024-11-25T12:05:43,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536403800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:43,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536403803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:43,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:43,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:43,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:43,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536403817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:43,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:43,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:43,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:43,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536403819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:43,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:43,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536403821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:43,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:43,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/1419a6fa05274e71aab4b1ab90e2883c is 50, key is test_row_0/B:col10/1732536343730/Put/seqid=0 2024-11-25T12:05:43,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742306_1482 (size=12151) 2024-11-25T12:05:43,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/1419a6fa05274e71aab4b1ab90e2883c 2024-11-25T12:05:43,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/1222c39929734542a66ac9b86e9cea83 is 50, key is test_row_0/C:col10/1732536343730/Put/seqid=0 2024-11-25T12:05:43,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536403926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:43,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:43,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536403926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:43,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:43,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:43,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:43,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536403936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:43,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:43,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536403947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:43,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536403947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:43,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742307_1483 (size=12151) 2024-11-25T12:05:43,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/1222c39929734542a66ac9b86e9cea83 2024-11-25T12:05:43,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/a327600ea4104b2f98a431b029627ae6 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/a327600ea4104b2f98a431b029627ae6 2024-11-25T12:05:43,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/a327600ea4104b2f98a431b029627ae6, entries=200, sequenceid=227, filesize=14.2 K 2024-11-25T12:05:43,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/1419a6fa05274e71aab4b1ab90e2883c as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/1419a6fa05274e71aab4b1ab90e2883c 2024-11-25T12:05:43,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/1419a6fa05274e71aab4b1ab90e2883c, entries=150, sequenceid=227, filesize=11.9 K 2024-11-25T12:05:43,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/1222c39929734542a66ac9b86e9cea83 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/1222c39929734542a66ac9b86e9cea83 2024-11-25T12:05:43,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/1222c39929734542a66ac9b86e9cea83, entries=150, sequenceid=227, filesize=11.9 K 2024-11-25T12:05:43,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d2665826b05855fc4d139a82044b59e6 in 241ms, sequenceid=227, compaction requested=true 2024-11-25T12:05:43,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:43,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:05:43,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:43,989 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:43,989 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:43,993 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39117 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:43,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:05:43,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:43,993 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): d2665826b05855fc4d139a82044b59e6/A is initiating minor compaction (all files) 2024-11-25T12:05:43,993 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2665826b05855fc4d139a82044b59e6/A in TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:43,993 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:43,993 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/a69ef55aba904ea9b5e6b1ec89eb981e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/deebb2ee24324349b6a0403ae5edd85f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/a327600ea4104b2f98a431b029627ae6] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp, totalSize=38.2 K 2024-11-25T12:05:43,993 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): d2665826b05855fc4d139a82044b59e6/B is initiating minor compaction (all files) 2024-11-25T12:05:43,993 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2665826b05855fc4d139a82044b59e6/B in TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:43,993 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/f8ab7a45d7d44dc296e071e6070f851e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/176f121b6a5c467e93356b672ce48be7, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/1419a6fa05274e71aab4b1ab90e2883c] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp, totalSize=35.9 K 2024-11-25T12:05:43,993 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting a69ef55aba904ea9b5e6b1ec89eb981e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732536341021 2024-11-25T12:05:43,993 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting f8ab7a45d7d44dc296e071e6070f851e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732536341021 2024-11-25T12:05:43,994 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting deebb2ee24324349b6a0403ae5edd85f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732536341432 2024-11-25T12:05:43,994 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 176f121b6a5c467e93356b672ce48be7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732536341432 2024-11-25T12:05:43,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:43,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:43,995 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting a327600ea4104b2f98a431b029627ae6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732536343730 2024-11-25T12:05:43,995 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 1419a6fa05274e71aab4b1ab90e2883c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732536343730 2024-11-25T12:05:44,008 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2665826b05855fc4d139a82044b59e6#B#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:44,008 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2665826b05855fc4d139a82044b59e6#A#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:44,009 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/21eeea05317b43f7aab17781cbee83da is 50, key is test_row_0/B:col10/1732536343730/Put/seqid=0 2024-11-25T12:05:44,009 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/baa5453f003642fd83d45fdc5c63daf7 is 50, key is test_row_0/A:col10/1732536343730/Put/seqid=0 2024-11-25T12:05:44,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742308_1484 (size=12527) 2024-11-25T12:05:44,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742309_1485 (size=12527) 2024-11-25T12:05:44,059 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/baa5453f003642fd83d45fdc5c63daf7 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/baa5453f003642fd83d45fdc5c63daf7 2024-11-25T12:05:44,066 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2665826b05855fc4d139a82044b59e6/A of d2665826b05855fc4d139a82044b59e6 into baa5453f003642fd83d45fdc5c63daf7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:44,067 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:44,067 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., storeName=d2665826b05855fc4d139a82044b59e6/A, priority=13, startTime=1732536343989; duration=0sec 2024-11-25T12:05:44,067 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:44,067 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:A 2024-11-25T12:05:44,067 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:44,068 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:44,068 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): d2665826b05855fc4d139a82044b59e6/C is initiating minor compaction (all files) 2024-11-25T12:05:44,068 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2665826b05855fc4d139a82044b59e6/C in TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:44,068 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/45969a05986a464ba2244cf40ae1ad16, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/c49c9a1b40f94bcf823ead26934f1cd0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/1222c39929734542a66ac9b86e9cea83] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp, totalSize=35.9 K 2024-11-25T12:05:44,068 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45969a05986a464ba2244cf40ae1ad16, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732536341021 2024-11-25T12:05:44,068 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting c49c9a1b40f94bcf823ead26934f1cd0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732536341432 2024-11-25T12:05:44,072 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1222c39929734542a66ac9b86e9cea83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732536343730 2024-11-25T12:05:44,080 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2665826b05855fc4d139a82044b59e6#C#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:44,080 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/289c3191d6414477a588995ee70dbbad is 50, key is test_row_0/C:col10/1732536343730/Put/seqid=0 2024-11-25T12:05:44,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742310_1486 (size=12527) 2024-11-25T12:05:44,111 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/289c3191d6414477a588995ee70dbbad as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/289c3191d6414477a588995ee70dbbad 2024-11-25T12:05:44,115 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2665826b05855fc4d139a82044b59e6/C of d2665826b05855fc4d139a82044b59e6 into 289c3191d6414477a588995ee70dbbad(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:44,115 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:44,115 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., storeName=d2665826b05855fc4d139a82044b59e6/C, priority=13, startTime=1732536343993; duration=0sec 2024-11-25T12:05:44,115 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:44,115 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:C 2024-11-25T12:05:44,149 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:44,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-25T12:05:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:44,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/209af21cdfc547db819ce76384b2f31f is 50, key is test_row_0/A:col10/1732536343799/Put/seqid=0 2024-11-25T12:05:44,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742311_1487 (size=14541) 2024-11-25T12:05:44,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536404174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:44,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536404193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:44,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:44,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536404195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:44,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:44,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536404197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:44,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536404198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:44,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:44,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:44,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536404306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:44,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:44,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536404320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:44,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536404320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:44,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536404320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:44,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536404321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:44,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:44,441 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/21eeea05317b43f7aab17781cbee83da as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/21eeea05317b43f7aab17781cbee83da 2024-11-25T12:05:44,451 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2665826b05855fc4d139a82044b59e6/B of d2665826b05855fc4d139a82044b59e6 into 21eeea05317b43f7aab17781cbee83da(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:44,451 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:44,451 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., storeName=d2665826b05855fc4d139a82044b59e6/B, priority=13, startTime=1732536343989; duration=0sec 2024-11-25T12:05:44,451 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:44,451 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:B 2024-11-25T12:05:44,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536404526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:44,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:44,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:44,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536404527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:44,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:44,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:44,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536404527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:44,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:44,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536404527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:44,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:44,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536404536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:44,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,549 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:44,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/209af21cdfc547db819ce76384b2f31f 2024-11-25T12:05:44,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/3a8c5919d8a0496a95330c02427fec16 is 50, key is test_row_0/B:col10/1732536343799/Put/seqid=0 2024-11-25T12:05:44,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742312_1488 (size=12151) 2024-11-25T12:05:44,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:44,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536404846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:44,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:44,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:44,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536404847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:44,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:44,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536404847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:44,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:44,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:44,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536404847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:44,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:44,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:44,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536404856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:44,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:44,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:45,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/3a8c5919d8a0496a95330c02427fec16 2024-11-25T12:05:45,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/97dda60b1ae74e9c9d1e9881efcddb99 is 50, key is test_row_0/C:col10/1732536343799/Put/seqid=0 2024-11-25T12:05:45,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742313_1489 (size=12151) 2024-11-25T12:05:45,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/97dda60b1ae74e9c9d1e9881efcddb99 2024-11-25T12:05:45,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/209af21cdfc547db819ce76384b2f31f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/209af21cdfc547db819ce76384b2f31f 2024-11-25T12:05:45,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/209af21cdfc547db819ce76384b2f31f, entries=200, sequenceid=248, filesize=14.2 K 2024-11-25T12:05:45,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/3a8c5919d8a0496a95330c02427fec16 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/3a8c5919d8a0496a95330c02427fec16 2024-11-25T12:05:45,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/3a8c5919d8a0496a95330c02427fec16, entries=150, sequenceid=248, filesize=11.9 K 2024-11-25T12:05:45,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/97dda60b1ae74e9c9d1e9881efcddb99 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/97dda60b1ae74e9c9d1e9881efcddb99 2024-11-25T12:05:45,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/97dda60b1ae74e9c9d1e9881efcddb99, entries=150, sequenceid=248, filesize=11.9 K 2024-11-25T12:05:45,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for d2665826b05855fc4d139a82044b59e6 in 921ms, sequenceid=248, compaction requested=false 2024-11-25T12:05:45,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:45,376 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:45,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-25T12:05:45,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:45,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/6192fe1c9a5e48b5942401976e6338c8 is 50, key is test_row_0/A:col10/1732536344188/Put/seqid=0 2024-11-25T12:05:45,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742314_1490 (size=12301) 2024-11-25T12:05:45,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:45,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536405436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:45,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536405449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:45,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:45,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536405450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:45,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:45,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:45,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536405452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:45,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536405453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:45,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:45,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536405556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:45,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536405567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:45,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:45,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536405568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:45,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:45,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536405577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:45,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536405577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:45,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-25T12:05:45,689 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:45,690 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:05:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-25T12:05:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T12:05:45,692 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:05:45,693 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:05:45,693 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:05:45,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536405777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:45,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:45,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536405777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:45,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:45,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536405777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:45,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:45,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T12:05:45,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:45,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536405806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:45,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:45,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536405806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:45,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:45,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:45,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/6192fe1c9a5e48b5942401976e6338c8 2024-11-25T12:05:45,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:05:45,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:45,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:45,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:45,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:45,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:45,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/f8e3c2095e794259b5174ef74a7507c4 is 50, key is test_row_0/B:col10/1732536344188/Put/seqid=0 2024-11-25T12:05:45,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742315_1491 (size=12301) 2024-11-25T12:05:46,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T12:05:46,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:05:46,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:46,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:46,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:46,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:46,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:46,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:46,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:46,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536406100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:46,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:46,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536406101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:46,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:46,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536406108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:46,114 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:46,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:46,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536406126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:46,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:46,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536406126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:46,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:05:46,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:46,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:46,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:46,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:46,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:46,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T12:05:46,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/f8e3c2095e794259b5174ef74a7507c4 2024-11-25T12:05:46,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:05:46,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:46,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:46,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:46,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:46,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:46,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/04e023a95f4547d1b9848031dcdc672b is 50, key is test_row_0/C:col10/1732536344188/Put/seqid=0 2024-11-25T12:05:46,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742316_1492 (size=12301) 2024-11-25T12:05:46,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/04e023a95f4547d1b9848031dcdc672b 2024-11-25T12:05:46,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/6192fe1c9a5e48b5942401976e6338c8 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/6192fe1c9a5e48b5942401976e6338c8 2024-11-25T12:05:46,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/6192fe1c9a5e48b5942401976e6338c8, entries=150, sequenceid=267, filesize=12.0 K 2024-11-25T12:05:46,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/f8e3c2095e794259b5174ef74a7507c4 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/f8e3c2095e794259b5174ef74a7507c4 2024-11-25T12:05:46,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:05:46,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/f8e3c2095e794259b5174ef74a7507c4, entries=150, sequenceid=267, filesize=12.0 K 2024-11-25T12:05:46,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:46,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:46,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:46,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:46,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:46,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/04e023a95f4547d1b9848031dcdc672b as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/04e023a95f4547d1b9848031dcdc672b 2024-11-25T12:05:46,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/04e023a95f4547d1b9848031dcdc672b, entries=150, sequenceid=267, filesize=12.0 K 2024-11-25T12:05:46,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d2665826b05855fc4d139a82044b59e6 in 1167ms, sequenceid=267, compaction requested=true 2024-11-25T12:05:46,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:46,544 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:46,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:05:46,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:46,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:05:46,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:46,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:46,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:05:46,544 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:46,545 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:46,545 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): d2665826b05855fc4d139a82044b59e6/A is initiating minor compaction (all files) 2024-11-25T12:05:46,545 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2665826b05855fc4d139a82044b59e6/A in TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:46,545 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/baa5453f003642fd83d45fdc5c63daf7, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/209af21cdfc547db819ce76384b2f31f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/6192fe1c9a5e48b5942401976e6338c8] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp, totalSize=38.4 K 2024-11-25T12:05:46,549 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting baa5453f003642fd83d45fdc5c63daf7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732536343730 2024-11-25T12:05:46,553 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36979 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:46,553 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): d2665826b05855fc4d139a82044b59e6/B is initiating minor compaction (all files) 2024-11-25T12:05:46,553 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2665826b05855fc4d139a82044b59e6/B in TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:46,553 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/21eeea05317b43f7aab17781cbee83da, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/3a8c5919d8a0496a95330c02427fec16, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/f8e3c2095e794259b5174ef74a7507c4] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp, totalSize=36.1 K 2024-11-25T12:05:46,553 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 209af21cdfc547db819ce76384b2f31f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732536343799 2024-11-25T12:05:46,553 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 21eeea05317b43f7aab17781cbee83da, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732536343730 2024-11-25T12:05:46,554 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6192fe1c9a5e48b5942401976e6338c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732536344188 2024-11-25T12:05:46,554 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a8c5919d8a0496a95330c02427fec16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732536343799 2024-11-25T12:05:46,554 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting f8e3c2095e794259b5174ef74a7507c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732536344188 2024-11-25T12:05:46,565 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2665826b05855fc4d139a82044b59e6#A#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:46,565 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/828483c3609047299fe0e19a43cc8deb is 50, key is test_row_0/A:col10/1732536344188/Put/seqid=0 2024-11-25T12:05:46,567 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2665826b05855fc4d139a82044b59e6#B#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:46,567 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/c518030dcd7745028a0ff4e2a93427cc is 50, key is test_row_0/B:col10/1732536344188/Put/seqid=0 2024-11-25T12:05:46,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742317_1493 (size=12779) 2024-11-25T12:05:46,603 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/828483c3609047299fe0e19a43cc8deb as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/828483c3609047299fe0e19a43cc8deb 2024-11-25T12:05:46,610 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2665826b05855fc4d139a82044b59e6/A of d2665826b05855fc4d139a82044b59e6 into 828483c3609047299fe0e19a43cc8deb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:46,610 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:46,610 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., storeName=d2665826b05855fc4d139a82044b59e6/A, priority=13, startTime=1732536346543; duration=0sec 2024-11-25T12:05:46,610 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:46,610 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:A 2024-11-25T12:05:46,611 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:46,612 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36979 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:46,612 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): d2665826b05855fc4d139a82044b59e6/C is initiating minor compaction (all files) 2024-11-25T12:05:46,612 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2665826b05855fc4d139a82044b59e6/C in TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:46,612 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/289c3191d6414477a588995ee70dbbad, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/97dda60b1ae74e9c9d1e9881efcddb99, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/04e023a95f4547d1b9848031dcdc672b] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp, totalSize=36.1 K 2024-11-25T12:05:46,618 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 289c3191d6414477a588995ee70dbbad, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732536343730 2024-11-25T12:05:46,622 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97dda60b1ae74e9c9d1e9881efcddb99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732536343799 2024-11-25T12:05:46,622 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04e023a95f4547d1b9848031dcdc672b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732536344188 2024-11-25T12:05:46,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742318_1494 (size=12779) 2024-11-25T12:05:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:46,637 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:46,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-25T12:05:46,645 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/c518030dcd7745028a0ff4e2a93427cc as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/c518030dcd7745028a0ff4e2a93427cc 2024-11-25T12:05:46,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/7aac50e16cd049c4b22209cae4902f03 is 50, key is test_row_0/A:col10/1732536345448/Put/seqid=0 2024-11-25T12:05:46,655 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2665826b05855fc4d139a82044b59e6/B of d2665826b05855fc4d139a82044b59e6 into c518030dcd7745028a0ff4e2a93427cc(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:46,655 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:46,655 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., storeName=d2665826b05855fc4d139a82044b59e6/B, priority=13, startTime=1732536346544; duration=0sec 2024-11-25T12:05:46,655 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:46,655 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:B 2024-11-25T12:05:46,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:05:46,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:46,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:46,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:46,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:46,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:46,659 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2665826b05855fc4d139a82044b59e6#C#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:46,660 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/a76bcdecdb5e4d05a68640ba58eeeb5e is 50, key is test_row_0/C:col10/1732536344188/Put/seqid=0 2024-11-25T12:05:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:46,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742319_1495 (size=12301) 2024-11-25T12:05:46,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742320_1496 (size=12779) 2024-11-25T12:05:46,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536406703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:46,722 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/a76bcdecdb5e4d05a68640ba58eeeb5e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/a76bcdecdb5e4d05a68640ba58eeeb5e 2024-11-25T12:05:46,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:46,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536406704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:46,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:46,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536406705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:46,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:46,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536406706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536406707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:46,728 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2665826b05855fc4d139a82044b59e6/C of d2665826b05855fc4d139a82044b59e6 into a76bcdecdb5e4d05a68640ba58eeeb5e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:46,728 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:46,728 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., storeName=d2665826b05855fc4d139a82044b59e6/C, priority=13, startTime=1732536346544; duration=0sec 2024-11-25T12:05:46,728 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:46,728 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:C 2024-11-25T12:05:46,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:05:46,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:46,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:46,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:46,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:46,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:46,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:46,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T12:05:46,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:46,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536406827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:46,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536406827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:46,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536406827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:46,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536406836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:46,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536406836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:46,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:05:46,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:46,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:46,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:46,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:46,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:46,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:47,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:47,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536407046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:47,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536407047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:47,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:47,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536407047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:47,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536407047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:47,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536407047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:47,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/7aac50e16cd049c4b22209cae4902f03 2024-11-25T12:05:47,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/d2fc910b93de4b3395cb6a52fda5e5f5 is 50, key is test_row_0/B:col10/1732536345448/Put/seqid=0 2024-11-25T12:05:47,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742321_1497 (size=12301) 2024-11-25T12:05:47,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/d2fc910b93de4b3395cb6a52fda5e5f5 2024-11-25T12:05:47,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/49b954ea2f5642b891c66ddefe38b2f3 is 50, key is test_row_0/C:col10/1732536345448/Put/seqid=0 2024-11-25T12:05:47,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:05:47,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:47,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:47,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:47,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:47,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:47,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742322_1498 (size=12301) 2024-11-25T12:05:47,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:05:47,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:47,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:47,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:47,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:47,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:47,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:47,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:47,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536407356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:47,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:47,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536407356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:47,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:47,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:47,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:47,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536407356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:47,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536407366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536407366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:47,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:47,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:05:47,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:47,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:47,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:47,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:47,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:47,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/49b954ea2f5642b891c66ddefe38b2f3 2024-11-25T12:05:47,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/7aac50e16cd049c4b22209cae4902f03 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/7aac50e16cd049c4b22209cae4902f03 2024-11-25T12:05:47,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/7aac50e16cd049c4b22209cae4902f03, entries=150, sequenceid=287, filesize=12.0 K 2024-11-25T12:05:47,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/d2fc910b93de4b3395cb6a52fda5e5f5 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/d2fc910b93de4b3395cb6a52fda5e5f5 2024-11-25T12:05:47,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/d2fc910b93de4b3395cb6a52fda5e5f5, entries=150, sequenceid=287, filesize=12.0 K 2024-11-25T12:05:47,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/49b954ea2f5642b891c66ddefe38b2f3 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/49b954ea2f5642b891c66ddefe38b2f3 2024-11-25T12:05:47,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/49b954ea2f5642b891c66ddefe38b2f3, entries=150, sequenceid=287, filesize=12.0 K 2024-11-25T12:05:47,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for d2665826b05855fc4d139a82044b59e6 in 957ms, sequenceid=287, compaction requested=false 2024-11-25T12:05:47,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:47,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-25T12:05:47,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:47,602 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-25T12:05:47,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/98e436b9469d44c0867f8d2bd27fcaa0 is 50, key is test_row_0/A:col10/1732536346705/Put/seqid=0 2024-11-25T12:05:47,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742323_1499 (size=12301) 2024-11-25T12:05:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T12:05:47,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:47,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536407938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:47,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536407940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:47,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536407942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536407944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:47,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536407952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:48,040 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/98e436b9469d44c0867f8d2bd27fcaa0 2024-11-25T12:05:48,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/7cd409a13c8c4a3ca39b9b11b522e965 is 50, key is test_row_0/B:col10/1732536346705/Put/seqid=0 2024-11-25T12:05:48,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742324_1500 (size=12301) 2024-11-25T12:05:48,070 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/7cd409a13c8c4a3ca39b9b11b522e965 2024-11-25T12:05:48,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/180da76e3e4f4daa8c5afdc7b8db7399 is 50, key is test_row_0/C:col10/1732536346705/Put/seqid=0 2024-11-25T12:05:48,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536408068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:48,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:48,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536408069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:48,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:48,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536408069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:48,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536408069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:48,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536408069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:48,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742325_1501 (size=12301) 2024-11-25T12:05:48,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:48,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536408287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:48,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:48,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536408296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536408296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:48,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:48,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536408296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536408296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,312 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:48,504 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/180da76e3e4f4daa8c5afdc7b8db7399 2024-11-25T12:05:48,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/98e436b9469d44c0867f8d2bd27fcaa0 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/98e436b9469d44c0867f8d2bd27fcaa0 2024-11-25T12:05:48,540 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/98e436b9469d44c0867f8d2bd27fcaa0, entries=150, sequenceid=306, filesize=12.0 K 2024-11-25T12:05:48,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/7cd409a13c8c4a3ca39b9b11b522e965 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/7cd409a13c8c4a3ca39b9b11b522e965 2024-11-25T12:05:48,569 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/7cd409a13c8c4a3ca39b9b11b522e965, entries=150, sequenceid=306, filesize=12.0 K 2024-11-25T12:05:48,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/180da76e3e4f4daa8c5afdc7b8db7399 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/180da76e3e4f4daa8c5afdc7b8db7399 2024-11-25T12:05:48,587 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/180da76e3e4f4daa8c5afdc7b8db7399, entries=150, sequenceid=306, filesize=12.0 K 2024-11-25T12:05:48,591 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for d2665826b05855fc4d139a82044b59e6 in 990ms, sequenceid=306, compaction requested=true 2024-11-25T12:05:48,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:48,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:48,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-25T12:05:48,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-25T12:05:48,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-25T12:05:48,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9010 sec 2024-11-25T12:05:48,599 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.9070 sec 2024-11-25T12:05:48,631 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:48,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-25T12:05:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:48,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/2786ca7b2c3d4e3b9c39840093cb2f39 is 50, key is test_row_0/A:col10/1732536348627/Put/seqid=0 2024-11-25T12:05:48,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742326_1502 (size=14741) 2024-11-25T12:05:48,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536408678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:48,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/2786ca7b2c3d4e3b9c39840093cb2f39 2024-11-25T12:05:48,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536408684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:48,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536408696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:48,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:48,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/65612db69e6b41ef883e7e34c64f228f is 50, key is test_row_0/B:col10/1732536348627/Put/seqid=0 2024-11-25T12:05:48,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:48,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536408708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:48,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:48,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536408709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:48,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742327_1503 (size=12301) 2024-11-25T12:05:48,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536408817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:48,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:48,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536408817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:48,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:48,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536408817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:48,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:48,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536408837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:48,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:48,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:48,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536408837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:48,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:48,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536409026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536409026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536409041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536409066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536409076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/65612db69e6b41ef883e7e34c64f228f 2024-11-25T12:05:49,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/b407ab934d5f4ee78fbd432eaa5ab8c1 is 50, key is test_row_0/C:col10/1732536348627/Put/seqid=0 2024-11-25T12:05:49,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742328_1504 (size=12301) 2024-11-25T12:05:49,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/b407ab934d5f4ee78fbd432eaa5ab8c1 2024-11-25T12:05:49,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/2786ca7b2c3d4e3b9c39840093cb2f39 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/2786ca7b2c3d4e3b9c39840093cb2f39 2024-11-25T12:05:49,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/2786ca7b2c3d4e3b9c39840093cb2f39, entries=200, sequenceid=326, filesize=14.4 K 2024-11-25T12:05:49,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/65612db69e6b41ef883e7e34c64f228f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/65612db69e6b41ef883e7e34c64f228f 2024-11-25T12:05:49,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/65612db69e6b41ef883e7e34c64f228f, entries=150, sequenceid=326, filesize=12.0 K 2024-11-25T12:05:49,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/b407ab934d5f4ee78fbd432eaa5ab8c1 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/b407ab934d5f4ee78fbd432eaa5ab8c1 2024-11-25T12:05:49,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/b407ab934d5f4ee78fbd432eaa5ab8c1, entries=150, sequenceid=326, filesize=12.0 K 2024-11-25T12:05:49,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for d2665826b05855fc4d139a82044b59e6 in 589ms, sequenceid=326, compaction requested=true 2024-11-25T12:05:49,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:49,220 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:49,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:05:49,221 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52122 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:49,221 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): d2665826b05855fc4d139a82044b59e6/A is initiating minor compaction (all files) 2024-11-25T12:05:49,221 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2665826b05855fc4d139a82044b59e6/A in TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:49,222 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/828483c3609047299fe0e19a43cc8deb, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/7aac50e16cd049c4b22209cae4902f03, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/98e436b9469d44c0867f8d2bd27fcaa0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/2786ca7b2c3d4e3b9c39840093cb2f39] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp, totalSize=50.9 K 2024-11-25T12:05:49,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:49,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:05:49,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:49,224 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 828483c3609047299fe0e19a43cc8deb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732536344188 2024-11-25T12:05:49,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:49,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:05:49,225 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:49,228 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aac50e16cd049c4b22209cae4902f03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732536345448 2024-11-25T12:05:49,229 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98e436b9469d44c0867f8d2bd27fcaa0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732536346682 2024-11-25T12:05:49,229 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:49,229 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): d2665826b05855fc4d139a82044b59e6/B is initiating minor compaction (all files) 2024-11-25T12:05:49,229 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2665826b05855fc4d139a82044b59e6/B in TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:49,229 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/c518030dcd7745028a0ff4e2a93427cc, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/d2fc910b93de4b3395cb6a52fda5e5f5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/7cd409a13c8c4a3ca39b9b11b522e965, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/65612db69e6b41ef883e7e34c64f228f] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp, totalSize=48.5 K 2024-11-25T12:05:49,229 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2786ca7b2c3d4e3b9c39840093cb2f39, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732536347916 2024-11-25T12:05:49,230 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting c518030dcd7745028a0ff4e2a93427cc, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732536344188 2024-11-25T12:05:49,230 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting d2fc910b93de4b3395cb6a52fda5e5f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732536345448 2024-11-25T12:05:49,230 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cd409a13c8c4a3ca39b9b11b522e965, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732536346682 2024-11-25T12:05:49,231 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 65612db69e6b41ef883e7e34c64f228f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732536347941 2024-11-25T12:05:49,259 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2665826b05855fc4d139a82044b59e6#B#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:49,260 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/5d992bb59c194aadb35e3f9095ffc435 is 50, key is test_row_0/B:col10/1732536348627/Put/seqid=0 2024-11-25T12:05:49,267 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2665826b05855fc4d139a82044b59e6#A#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:49,268 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/cda0c18943d34cf7ac0a11f2553bd4b8 is 50, key is test_row_0/A:col10/1732536348627/Put/seqid=0 2024-11-25T12:05:49,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742329_1505 (size=12915) 2024-11-25T12:05:49,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742330_1506 (size=12915) 2024-11-25T12:05:49,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:49,356 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:49,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-25T12:05:49,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/e5dfa841ae5d4524b33027571cd71b13 is 50, key is test_row_0/A:col10/1732536349353/Put/seqid=0 2024-11-25T12:05:49,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742331_1507 (size=17181) 2024-11-25T12:05:49,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/e5dfa841ae5d4524b33027571cd71b13 2024-11-25T12:05:49,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/3b01ff2013e245e886ea991e5c68535b is 50, key is test_row_0/B:col10/1732536349353/Put/seqid=0 2024-11-25T12:05:49,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742332_1508 (size=12301) 2024-11-25T12:05:49,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/3b01ff2013e245e886ea991e5c68535b 2024-11-25T12:05:49,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/749f1dc3ce264469830e1df7147c2f9d is 50, key is test_row_0/C:col10/1732536349353/Put/seqid=0 2024-11-25T12:05:49,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742333_1509 (size=12301) 2024-11-25T12:05:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536409412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/749f1dc3ce264469830e1df7147c2f9d 2024-11-25T12:05:49,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536409417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536409418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536409418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536409422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/e5dfa841ae5d4524b33027571cd71b13 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/e5dfa841ae5d4524b33027571cd71b13 2024-11-25T12:05:49,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/e5dfa841ae5d4524b33027571cd71b13, entries=250, sequenceid=345, filesize=16.8 K 2024-11-25T12:05:49,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/3b01ff2013e245e886ea991e5c68535b as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/3b01ff2013e245e886ea991e5c68535b 2024-11-25T12:05:49,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/3b01ff2013e245e886ea991e5c68535b, entries=150, sequenceid=345, filesize=12.0 K 2024-11-25T12:05:49,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/749f1dc3ce264469830e1df7147c2f9d as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/749f1dc3ce264469830e1df7147c2f9d 2024-11-25T12:05:49,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/749f1dc3ce264469830e1df7147c2f9d, entries=150, sequenceid=345, filesize=12.0 K 2024-11-25T12:05:49,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for d2665826b05855fc4d139a82044b59e6 in 103ms, sequenceid=345, compaction requested=true 2024-11-25T12:05:49,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:49,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:A, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:49,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:05:49,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:B, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:49,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:05:49,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:49,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-25T12:05:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:49,546 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:49,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-25T12:05:49,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/2459c844297941299bd88c4038a58a56 is 50, key is test_row_0/A:col10/1732536349544/Put/seqid=0 2024-11-25T12:05:49,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742334_1510 (size=17181) 2024-11-25T12:05:49,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/2459c844297941299bd88c4038a58a56 2024-11-25T12:05:49,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/83feecf28d6d4aaaa0cdbe4e5e1c4f2d is 50, key is test_row_0/B:col10/1732536349544/Put/seqid=0 2024-11-25T12:05:49,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742335_1511 (size=12301) 2024-11-25T12:05:49,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/83feecf28d6d4aaaa0cdbe4e5e1c4f2d 2024-11-25T12:05:49,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/73e43962bcf24bbfbac13363ab828001 is 50, key is test_row_0/C:col10/1732536349544/Put/seqid=0 2024-11-25T12:05:49,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536409611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536409611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742336_1512 (size=12301) 2024-11-25T12:05:49,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/73e43962bcf24bbfbac13363ab828001 2024-11-25T12:05:49,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/2459c844297941299bd88c4038a58a56 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/2459c844297941299bd88c4038a58a56 2024-11-25T12:05:49,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/2459c844297941299bd88c4038a58a56, entries=250, sequenceid=364, filesize=16.8 K 2024-11-25T12:05:49,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/83feecf28d6d4aaaa0cdbe4e5e1c4f2d as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/83feecf28d6d4aaaa0cdbe4e5e1c4f2d 2024-11-25T12:05:49,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/83feecf28d6d4aaaa0cdbe4e5e1c4f2d, entries=150, sequenceid=364, filesize=12.0 K 2024-11-25T12:05:49,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/73e43962bcf24bbfbac13363ab828001 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/73e43962bcf24bbfbac13363ab828001 2024-11-25T12:05:49,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536409630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536409634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536409636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/73e43962bcf24bbfbac13363ab828001, entries=150, sequenceid=364, filesize=12.0 K 2024-11-25T12:05:49,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for d2665826b05855fc4d139a82044b59e6 in 122ms, sequenceid=364, compaction requested=true 2024-11-25T12:05:49,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:49,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:A, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:49,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=5), splitQueue=0 2024-11-25T12:05:49,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:B, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:49,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=6), splitQueue=0 2024-11-25T12:05:49,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2665826b05855fc4d139a82044b59e6:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:49,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=7), splitQueue=0 2024-11-25T12:05:49,713 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/5d992bb59c194aadb35e3f9095ffc435 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/5d992bb59c194aadb35e3f9095ffc435 2024-11-25T12:05:49,724 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/cda0c18943d34cf7ac0a11f2553bd4b8 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/cda0c18943d34cf7ac0a11f2553bd4b8 2024-11-25T12:05:49,734 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2665826b05855fc4d139a82044b59e6/A of d2665826b05855fc4d139a82044b59e6 into cda0c18943d34cf7ac0a11f2553bd4b8(size=12.6 K), total size for store is 46.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:49,734 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:49,734 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., storeName=d2665826b05855fc4d139a82044b59e6/A, priority=12, startTime=1732536349220; duration=0sec 2024-11-25T12:05:49,734 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=7), splitQueue=0 2024-11-25T12:05:49,734 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:A 2024-11-25T12:05:49,734 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:A 2024-11-25T12:05:49,734 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-25T12:05:49,734 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2665826b05855fc4d139a82044b59e6/B of d2665826b05855fc4d139a82044b59e6 into 5d992bb59c194aadb35e3f9095ffc435(size=12.6 K), total size for store is 36.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:49,734 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:49,735 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., storeName=d2665826b05855fc4d139a82044b59e6/B, priority=12, startTime=1732536349224; duration=0sec 2024-11-25T12:05:49,735 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=6), splitQueue=0 2024-11-25T12:05:49,735 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:B 2024-11-25T12:05:49,735 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:B 2024-11-25T12:05:49,735 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:A 2024-11-25T12:05:49,739 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74284 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-25T12:05:49,739 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): d2665826b05855fc4d139a82044b59e6/C is initiating minor compaction (all files) 2024-11-25T12:05:49,740 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2665826b05855fc4d139a82044b59e6/C in TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:49,740 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/a76bcdecdb5e4d05a68640ba58eeeb5e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/49b954ea2f5642b891c66ddefe38b2f3, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/180da76e3e4f4daa8c5afdc7b8db7399, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/b407ab934d5f4ee78fbd432eaa5ab8c1, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/749f1dc3ce264469830e1df7147c2f9d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/73e43962bcf24bbfbac13363ab828001] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp, totalSize=72.5 K 2024-11-25T12:05:49,740 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 6 compacting, 0 eligible, 16 blocking 2024-11-25T12:05:49,740 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T12:05:49,740 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T12:05:49,740 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. because compaction request was cancelled 2024-11-25T12:05:49,740 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:C 2024-11-25T12:05:49,740 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:B 2024-11-25T12:05:49,740 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 6 compacting, 0 eligible, 16 blocking 2024-11-25T12:05:49,740 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T12:05:49,740 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T12:05:49,740 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. because compaction request was cancelled 2024-11-25T12:05:49,740 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:C 2024-11-25T12:05:49,740 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:49,741 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting a76bcdecdb5e4d05a68640ba58eeeb5e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732536344188 2024-11-25T12:05:49,742 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:49,742 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): d2665826b05855fc4d139a82044b59e6/B is initiating minor compaction (all files) 2024-11-25T12:05:49,742 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2665826b05855fc4d139a82044b59e6/B in TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:49,742 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/5d992bb59c194aadb35e3f9095ffc435, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/3b01ff2013e245e886ea991e5c68535b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/83feecf28d6d4aaaa0cdbe4e5e1c4f2d] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp, totalSize=36.6 K 2024-11-25T12:05:49,742 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49b954ea2f5642b891c66ddefe38b2f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732536345448 2024-11-25T12:05:49,743 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 180da76e3e4f4daa8c5afdc7b8db7399, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1732536346682 2024-11-25T12:05:49,743 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d992bb59c194aadb35e3f9095ffc435, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732536347941 2024-11-25T12:05:49,743 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting b407ab934d5f4ee78fbd432eaa5ab8c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732536347941 2024-11-25T12:05:49,743 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b01ff2013e245e886ea991e5c68535b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732536348694 2024-11-25T12:05:49,744 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 83feecf28d6d4aaaa0cdbe4e5e1c4f2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732536349415 2024-11-25T12:05:49,744 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 749f1dc3ce264469830e1df7147c2f9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732536348694 2024-11-25T12:05:49,745 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73e43962bcf24bbfbac13363ab828001, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732536349415 2024-11-25T12:05:49,752 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2665826b05855fc4d139a82044b59e6#B#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:49,753 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/4949ef1a5281481a81e648413e130630 is 50, key is test_row_0/B:col10/1732536349544/Put/seqid=0 2024-11-25T12:05:49,756 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2665826b05855fc4d139a82044b59e6#C#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:49,757 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/59cda2bc502c44a7a9f8322655d879d6 is 50, key is test_row_0/C:col10/1732536349544/Put/seqid=0 2024-11-25T12:05:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:49,769 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:49,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-25T12:05:49,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742337_1513 (size=13017) 2024-11-25T12:05:49,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/6576c543b0d1473ca5d1f18eeadf3d1b is 50, key is test_row_0/A:col10/1732536349767/Put/seqid=0 2024-11-25T12:05:49,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742338_1514 (size=12983) 2024-11-25T12:05:49,797 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/4949ef1a5281481a81e648413e130630 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/4949ef1a5281481a81e648413e130630 2024-11-25T12:05:49,801 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2665826b05855fc4d139a82044b59e6/B of d2665826b05855fc4d139a82044b59e6 into 4949ef1a5281481a81e648413e130630(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:49,801 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:49,801 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., storeName=d2665826b05855fc4d139a82044b59e6/B, priority=13, startTime=1732536349459; duration=0sec 2024-11-25T12:05:49,802 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:05:49,802 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:B 2024-11-25T12:05:49,802 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:B 2024-11-25T12:05:49,802 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:05:49,803 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:05:49,803 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): d2665826b05855fc4d139a82044b59e6/A is initiating minor compaction (all files) 2024-11-25T12:05:49,803 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2665826b05855fc4d139a82044b59e6/A in TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:49,803 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/cda0c18943d34cf7ac0a11f2553bd4b8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/e5dfa841ae5d4524b33027571cd71b13, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/2459c844297941299bd88c4038a58a56] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp, totalSize=46.2 K 2024-11-25T12:05:49,804 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting cda0c18943d34cf7ac0a11f2553bd4b8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732536347941 2024-11-25T12:05:49,804 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting e5dfa841ae5d4524b33027571cd71b13, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732536348667 2024-11-25T12:05:49,804 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 2459c844297941299bd88c4038a58a56, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1732536349411 2024-11-25T12:05:49,820 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2665826b05855fc4d139a82044b59e6#A#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:49,821 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/42279b2fe06446c19a5bab4d7eff8540 is 50, key is test_row_0/A:col10/1732536349544/Put/seqid=0 2024-11-25T12:05:49,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742339_1515 (size=12301) 2024-11-25T12:05:49,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/6576c543b0d1473ca5d1f18eeadf3d1b 2024-11-25T12:05:49,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/b68ce32585034767b7dd7f51302a4bd4 is 50, key is test_row_0/B:col10/1732536349767/Put/seqid=0 2024-11-25T12:05:49,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536409829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742340_1516 (size=13017) 2024-11-25T12:05:49,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536409836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536409846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-25T12:05:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536409847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,869 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/42279b2fe06446c19a5bab4d7eff8540 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/42279b2fe06446c19a5bab4d7eff8540 2024-11-25T12:05:49,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536409848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,873 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:49,879 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:05:49,881 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2665826b05855fc4d139a82044b59e6/A of d2665826b05855fc4d139a82044b59e6 into 42279b2fe06446c19a5bab4d7eff8540(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:49,882 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:49,882 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., storeName=d2665826b05855fc4d139a82044b59e6/A, priority=13, startTime=1732536349459; duration=0sec 2024-11-25T12:05:49,882 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:05:49,882 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:A 2024-11-25T12:05:49,882 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:A 2024-11-25T12:05:49,882 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-25T12:05:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-25T12:05:49,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T12:05:49,885 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:05:49,888 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T12:05:49,889 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T12:05:49,889 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. because compaction request was cancelled 2024-11-25T12:05:49,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742341_1517 (size=12301) 2024-11-25T12:05:49,889 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:B 2024-11-25T12:05:49,889 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-25T12:05:49,889 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T12:05:49,889 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T12:05:49,889 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. because compaction request was cancelled 2024-11-25T12:05:49,889 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:A 2024-11-25T12:05:49,892 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:05:49,893 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:05:49,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536409960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536409966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536409977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536409977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536409977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:49,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T12:05:50,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T12:05:50,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:50,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:50,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:50,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:50,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:50,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:50,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536410186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:50,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:50,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536410186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:50,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:50,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536410197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:50,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:50,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:50,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:50,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:50,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536410197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:50,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:50,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:50,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:50,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T12:05:50,209 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/59cda2bc502c44a7a9f8322655d879d6 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/59cda2bc502c44a7a9f8322655d879d6 2024-11-25T12:05:50,214 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in d2665826b05855fc4d139a82044b59e6/C of d2665826b05855fc4d139a82044b59e6 into 59cda2bc502c44a7a9f8322655d879d6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:05:50,214 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:50,215 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., storeName=d2665826b05855fc4d139a82044b59e6/C, priority=10, startTime=1732536349669; duration=0sec 2024-11-25T12:05:50,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:50,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536410200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:50,215 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:50,215 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2665826b05855fc4d139a82044b59e6:C 2024-11-25T12:05:50,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:50,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:50,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:50,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T12:05:50,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:50,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:50,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:50,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:50,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:50,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:50,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/b68ce32585034767b7dd7f51302a4bd4 2024-11-25T12:05:50,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/a268e813d240429db0ed956255c0c27e is 50, key is test_row_0/C:col10/1732536349767/Put/seqid=0 2024-11-25T12:05:50,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742342_1518 (size=12301) 2024-11-25T12:05:50,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T12:05:50,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:50,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:50,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:50,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:50,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:50,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T12:05:50,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:50,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40766 deadline: 1732536410506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:50,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:50,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40750 deadline: 1732536410508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:50,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:50,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40820 deadline: 1732536410516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:50,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40744 deadline: 1732536410516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:50,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:50,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:50,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:50,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:50,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:50,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:50,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:50,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40786 deadline: 1732536410527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:50,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 , the old value is region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:50,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:50,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., hostname=2bf237fd86ca,37643,1732536214674, seqNum=2 because the exception is null or not the one we care about 2024-11-25T12:05:50,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T12:05:50,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:50,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:50,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:50,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:50,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:50,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:50,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T12:05:50,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:50,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:50,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:50,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:50,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:50,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:50,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/a268e813d240429db0ed956255c0c27e 2024-11-25T12:05:50,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/6576c543b0d1473ca5d1f18eeadf3d1b as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/6576c543b0d1473ca5d1f18eeadf3d1b 2024-11-25T12:05:50,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/6576c543b0d1473ca5d1f18eeadf3d1b, entries=150, sequenceid=385, filesize=12.0 K 2024-11-25T12:05:50,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/b68ce32585034767b7dd7f51302a4bd4 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/b68ce32585034767b7dd7f51302a4bd4 2024-11-25T12:05:50,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/b68ce32585034767b7dd7f51302a4bd4, entries=150, sequenceid=385, filesize=12.0 K 2024-11-25T12:05:50,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/a268e813d240429db0ed956255c0c27e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/a268e813d240429db0ed956255c0c27e 2024-11-25T12:05:50,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/a268e813d240429db0ed956255c0c27e, entries=150, sequenceid=385, filesize=12.0 K 2024-11-25T12:05:50,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for d2665826b05855fc4d139a82044b59e6 in 1021ms, sequenceid=385, compaction requested=false 2024-11-25T12:05:50,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:50,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-25T12:05:50,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:50,897 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-25T12:05:50,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/1c7d751f90ca42cc870b0501f593083a is 50, key is test_row_0/A:col10/1732536349842/Put/seqid=0 2024-11-25T12:05:50,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742343_1519 (size=12301) 2024-11-25T12:05:50,957 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/1c7d751f90ca42cc870b0501f593083a 2024-11-25T12:05:50,997 INFO [Thread-1987 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1987. 2024-11-25T12:05:50,997 DEBUG [Thread-1987 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:50,997 DEBUG [Thread-1987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:50,997 DEBUG [Thread-1987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:50,997 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:51,002 INFO [Thread-1981 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1981. 2024-11-25T12:05:51,002 DEBUG [Thread-1981 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:51,002 DEBUG [Thread-1981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:51,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/16611b9f4a0b4284afeb7a5a82431b88 is 50, key is test_row_0/B:col10/1732536349842/Put/seqid=0 2024-11-25T12:05:51,003 DEBUG [Thread-1981 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:51,005 INFO [Thread-1983 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1983. 2024-11-25T12:05:51,005 DEBUG [Thread-1983 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:51,005 DEBUG [Thread-1983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:51,005 DEBUG [Thread-1983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:51,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:51,007 INFO [Thread-1985 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1985. 2024-11-25T12:05:51,007 DEBUG [Thread-1985 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:51,007 DEBUG [Thread-1985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:51,007 DEBUG [Thread-1985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:51,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:51,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:51,008 INFO [Thread-1979 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1979. 2024-11-25T12:05:51,008 DEBUG [Thread-1979 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:51,008 DEBUG [Thread-1979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:51,008 DEBUG [Thread-1979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:51,008 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:51,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742344_1520 (size=12301) 2024-11-25T12:05:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T12:05:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:51,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. as already flushing 2024-11-25T12:05:51,027 INFO [Thread-1970 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1970. 2024-11-25T12:05:51,027 INFO [Thread-1974 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1974. 2024-11-25T12:05:51,027 DEBUG [Thread-1970 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:51,027 DEBUG [Thread-1974 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:51,027 DEBUG [Thread-1970 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:51,027 DEBUG [Thread-1974 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:51,028 DEBUG [Thread-1974 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:51,028 DEBUG [Thread-1970 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:51,028 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:51,028 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:51,038 INFO [Thread-1968 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1968. 2024-11-25T12:05:51,038 INFO [Thread-1972 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1972. 2024-11-25T12:05:51,038 DEBUG [Thread-1972 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:51,038 DEBUG [Thread-1968 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:51,038 DEBUG [Thread-1972 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:51,038 DEBUG [Thread-1968 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:51,038 DEBUG [Thread-1968 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:51,038 DEBUG [Thread-1972 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:51,038 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:51,038 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:51,048 INFO [Thread-1976 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1976. 2024-11-25T12:05:51,048 DEBUG [Thread-1976 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:05:51,048 DEBUG [Thread-1976 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:51,048 DEBUG [Thread-1976 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:51,048 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:51,413 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/16611b9f4a0b4284afeb7a5a82431b88 2024-11-25T12:05:51,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/f8970177ebe440049d72f4387e2080f2 is 50, key is test_row_0/C:col10/1732536349842/Put/seqid=0 2024-11-25T12:05:51,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742345_1521 (size=12301) 2024-11-25T12:05:51,826 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/f8970177ebe440049d72f4387e2080f2 2024-11-25T12:05:51,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/1c7d751f90ca42cc870b0501f593083a as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/1c7d751f90ca42cc870b0501f593083a 2024-11-25T12:05:51,834 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/1c7d751f90ca42cc870b0501f593083a, entries=150, sequenceid=405, filesize=12.0 K 2024-11-25T12:05:51,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/16611b9f4a0b4284afeb7a5a82431b88 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/16611b9f4a0b4284afeb7a5a82431b88 2024-11-25T12:05:51,838 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/16611b9f4a0b4284afeb7a5a82431b88, entries=150, sequenceid=405, filesize=12.0 K 2024-11-25T12:05:51,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/f8970177ebe440049d72f4387e2080f2 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/f8970177ebe440049d72f4387e2080f2 2024-11-25T12:05:51,843 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/f8970177ebe440049d72f4387e2080f2, entries=150, sequenceid=405, filesize=12.0 K 2024-11-25T12:05:51,844 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=33.54 KB/34350 for d2665826b05855fc4d139a82044b59e6 in 947ms, sequenceid=405, compaction requested=true 2024-11-25T12:05:51,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for d2665826b05855fc4d139a82044b59e6: 2024-11-25T12:05:51,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:51,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-25T12:05:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-25T12:05:51,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-25T12:05:51,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9530 sec 2024-11-25T12:05:51,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.9680 sec 2024-11-25T12:05:52,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-25T12:05:52,047 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:52,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T12:05:52,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-25T12:05:52,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-25T12:05:52,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-25T12:05:52,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-25T12:05:52,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-25T12:05:52,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T12:05:52,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T12:05:52,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 780 2024-11-25T12:05:52,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2340 rows 2024-11-25T12:05:52,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 781 2024-11-25T12:05:52,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2343 rows 2024-11-25T12:05:52,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 785 2024-11-25T12:05:52,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2351 rows 2024-11-25T12:05:52,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 785 2024-11-25T12:05:52,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2355 rows 2024-11-25T12:05:52,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 783 2024-11-25T12:05:52,047 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2349 rows 2024-11-25T12:05:52,048 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T12:05:52,048 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:05:52,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:52,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:52,048 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:05:52,049 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-25T12:05:52,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:52,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-25T12:05:52,052 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536352052"}]},"ts":"1732536352052"} 2024-11-25T12:05:52,054 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T12:05:52,054 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T12:05:52,055 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T12:05:52,057 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2665826b05855fc4d139a82044b59e6, UNASSIGN}] 2024-11-25T12:05:52,058 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2665826b05855fc4d139a82044b59e6, UNASSIGN 2024-11-25T12:05:52,058 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=d2665826b05855fc4d139a82044b59e6, regionState=CLOSING, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:52,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2665826b05855fc4d139a82044b59e6, UNASSIGN because future has completed 2024-11-25T12:05:52,060 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T12:05:52,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674}] 2024-11-25T12:05:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-25T12:05:52,213 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(122): Close d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:52,213 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T12:05:52,213 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1722): Closing d2665826b05855fc4d139a82044b59e6, disabling compactions & flushes 2024-11-25T12:05:52,213 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:52,213 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:52,213 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. after waiting 0 ms 2024-11-25T12:05:52,213 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:52,213 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(2902): Flushing d2665826b05855fc4d139a82044b59e6 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-25T12:05:52,218 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/cd5e646805a44c9fb0a5ca6ec92f2104 is 50, key is test_row_0/A:col10/1732536351036/Put/seqid=0 2024-11-25T12:05:52,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742346_1522 (size=12301) 2024-11-25T12:05:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-25T12:05:52,622 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/cd5e646805a44c9fb0a5ca6ec92f2104 2024-11-25T12:05:52,630 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/ffef066ad09c43199d038e1f3c7aca6f is 50, key is test_row_0/B:col10/1732536351036/Put/seqid=0 2024-11-25T12:05:52,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742347_1523 (size=12301) 2024-11-25T12:05:52,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-25T12:05:53,034 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/ffef066ad09c43199d038e1f3c7aca6f 2024-11-25T12:05:53,041 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/2e6eb9093845425c8246b9d6dfe08437 is 50, key is test_row_0/C:col10/1732536351036/Put/seqid=0 2024-11-25T12:05:53,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742348_1524 (size=12301) 2024-11-25T12:05:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-25T12:05:53,445 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/2e6eb9093845425c8246b9d6dfe08437 2024-11-25T12:05:53,449 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/A/cd5e646805a44c9fb0a5ca6ec92f2104 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/cd5e646805a44c9fb0a5ca6ec92f2104 2024-11-25T12:05:53,452 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/cd5e646805a44c9fb0a5ca6ec92f2104, entries=150, sequenceid=413, filesize=12.0 K 2024-11-25T12:05:53,453 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/B/ffef066ad09c43199d038e1f3c7aca6f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/ffef066ad09c43199d038e1f3c7aca6f 2024-11-25T12:05:53,456 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/ffef066ad09c43199d038e1f3c7aca6f, entries=150, sequenceid=413, filesize=12.0 K 2024-11-25T12:05:53,457 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/.tmp/C/2e6eb9093845425c8246b9d6dfe08437 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/2e6eb9093845425c8246b9d6dfe08437 2024-11-25T12:05:53,460 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/2e6eb9093845425c8246b9d6dfe08437, entries=150, sequenceid=413, filesize=12.0 K 2024-11-25T12:05:53,461 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for d2665826b05855fc4d139a82044b59e6 in 1248ms, sequenceid=413, compaction requested=true 2024-11-25T12:05:53,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/4c3713cdf5e3446bad6b88b8034be565, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/d9598553862341b28685c8cdbcd67a6d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/54591a2814ef4246a82e0f9f2917b647, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/580b092675824376ae286c5c7dea9d8f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/57d3ab8d918847f8b347eadfffdb6305, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/497f499bd9494631ae2f692cc4f1a212, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/203d262762da46a9b661a1b894d77f57, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/170c1a43ab264053a4f62d001c1f768b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/a69ef55aba904ea9b5e6b1ec89eb981e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/deebb2ee24324349b6a0403ae5edd85f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/a327600ea4104b2f98a431b029627ae6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/baa5453f003642fd83d45fdc5c63daf7, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/209af21cdfc547db819ce76384b2f31f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/828483c3609047299fe0e19a43cc8deb, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/6192fe1c9a5e48b5942401976e6338c8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/7aac50e16cd049c4b22209cae4902f03, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/98e436b9469d44c0867f8d2bd27fcaa0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/2786ca7b2c3d4e3b9c39840093cb2f39, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/cda0c18943d34cf7ac0a11f2553bd4b8, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/e5dfa841ae5d4524b33027571cd71b13, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/2459c844297941299bd88c4038a58a56] to archive 2024-11-25T12:05:53,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:05:53,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/4c3713cdf5e3446bad6b88b8034be565 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/4c3713cdf5e3446bad6b88b8034be565 2024-11-25T12:05:53,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/d9598553862341b28685c8cdbcd67a6d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/d9598553862341b28685c8cdbcd67a6d 2024-11-25T12:05:53,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/54591a2814ef4246a82e0f9f2917b647 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/54591a2814ef4246a82e0f9f2917b647 2024-11-25T12:05:53,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/580b092675824376ae286c5c7dea9d8f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/580b092675824376ae286c5c7dea9d8f 2024-11-25T12:05:53,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/57d3ab8d918847f8b347eadfffdb6305 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/57d3ab8d918847f8b347eadfffdb6305 2024-11-25T12:05:53,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/497f499bd9494631ae2f692cc4f1a212 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/497f499bd9494631ae2f692cc4f1a212 2024-11-25T12:05:53,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/203d262762da46a9b661a1b894d77f57 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/203d262762da46a9b661a1b894d77f57 2024-11-25T12:05:53,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/170c1a43ab264053a4f62d001c1f768b to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/170c1a43ab264053a4f62d001c1f768b 2024-11-25T12:05:53,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/a69ef55aba904ea9b5e6b1ec89eb981e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/a69ef55aba904ea9b5e6b1ec89eb981e 2024-11-25T12:05:53,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/deebb2ee24324349b6a0403ae5edd85f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/deebb2ee24324349b6a0403ae5edd85f 2024-11-25T12:05:53,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/a327600ea4104b2f98a431b029627ae6 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/a327600ea4104b2f98a431b029627ae6 2024-11-25T12:05:53,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/baa5453f003642fd83d45fdc5c63daf7 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/baa5453f003642fd83d45fdc5c63daf7 2024-11-25T12:05:53,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/209af21cdfc547db819ce76384b2f31f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/209af21cdfc547db819ce76384b2f31f 2024-11-25T12:05:53,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/828483c3609047299fe0e19a43cc8deb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/828483c3609047299fe0e19a43cc8deb 2024-11-25T12:05:53,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/6192fe1c9a5e48b5942401976e6338c8 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/6192fe1c9a5e48b5942401976e6338c8 2024-11-25T12:05:53,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/7aac50e16cd049c4b22209cae4902f03 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/7aac50e16cd049c4b22209cae4902f03 2024-11-25T12:05:53,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/98e436b9469d44c0867f8d2bd27fcaa0 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/98e436b9469d44c0867f8d2bd27fcaa0 2024-11-25T12:05:53,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/2786ca7b2c3d4e3b9c39840093cb2f39 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/2786ca7b2c3d4e3b9c39840093cb2f39 2024-11-25T12:05:53,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/cda0c18943d34cf7ac0a11f2553bd4b8 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/cda0c18943d34cf7ac0a11f2553bd4b8 2024-11-25T12:05:53,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/e5dfa841ae5d4524b33027571cd71b13 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/e5dfa841ae5d4524b33027571cd71b13 2024-11-25T12:05:53,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/2459c844297941299bd88c4038a58a56 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/2459c844297941299bd88c4038a58a56 2024-11-25T12:05:53,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/65ae3a0265d54be3802e5a3de9ffae96, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/c1b212d932884cf98ee2155f72aeb60a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/7282706a130548a48cb354431b9c4b88, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/19a1c8ca8ce948cfaec4e1c7d043317a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/c14238af66604261ae9ef0443ce05730, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/5ee9d451b4c94d459986222b3ccc5cce, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/423f7a8cd9f641d8b7514066c44a7b97, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/f8ab7a45d7d44dc296e071e6070f851e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/2e1d1ba51b754aa290c4e430796ff0f3, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/176f121b6a5c467e93356b672ce48be7, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/21eeea05317b43f7aab17781cbee83da, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/1419a6fa05274e71aab4b1ab90e2883c, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/3a8c5919d8a0496a95330c02427fec16, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/c518030dcd7745028a0ff4e2a93427cc, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/f8e3c2095e794259b5174ef74a7507c4, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/d2fc910b93de4b3395cb6a52fda5e5f5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/7cd409a13c8c4a3ca39b9b11b522e965, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/5d992bb59c194aadb35e3f9095ffc435, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/65612db69e6b41ef883e7e34c64f228f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/3b01ff2013e245e886ea991e5c68535b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/83feecf28d6d4aaaa0cdbe4e5e1c4f2d] to archive 2024-11-25T12:05:53,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:05:53,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/65ae3a0265d54be3802e5a3de9ffae96 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/65ae3a0265d54be3802e5a3de9ffae96 2024-11-25T12:05:53,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/c1b212d932884cf98ee2155f72aeb60a to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/c1b212d932884cf98ee2155f72aeb60a 2024-11-25T12:05:53,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/7282706a130548a48cb354431b9c4b88 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/7282706a130548a48cb354431b9c4b88 2024-11-25T12:05:53,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/19a1c8ca8ce948cfaec4e1c7d043317a to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/19a1c8ca8ce948cfaec4e1c7d043317a 2024-11-25T12:05:53,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/c14238af66604261ae9ef0443ce05730 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/c14238af66604261ae9ef0443ce05730 2024-11-25T12:05:53,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/5ee9d451b4c94d459986222b3ccc5cce to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/5ee9d451b4c94d459986222b3ccc5cce 2024-11-25T12:05:53,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/423f7a8cd9f641d8b7514066c44a7b97 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/423f7a8cd9f641d8b7514066c44a7b97 2024-11-25T12:05:53,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/f8ab7a45d7d44dc296e071e6070f851e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/f8ab7a45d7d44dc296e071e6070f851e 2024-11-25T12:05:53,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/2e1d1ba51b754aa290c4e430796ff0f3 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/2e1d1ba51b754aa290c4e430796ff0f3 2024-11-25T12:05:53,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/176f121b6a5c467e93356b672ce48be7 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/176f121b6a5c467e93356b672ce48be7 2024-11-25T12:05:53,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/21eeea05317b43f7aab17781cbee83da to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/21eeea05317b43f7aab17781cbee83da 2024-11-25T12:05:53,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/1419a6fa05274e71aab4b1ab90e2883c to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/1419a6fa05274e71aab4b1ab90e2883c 2024-11-25T12:05:53,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/3a8c5919d8a0496a95330c02427fec16 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/3a8c5919d8a0496a95330c02427fec16 2024-11-25T12:05:53,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/c518030dcd7745028a0ff4e2a93427cc to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/c518030dcd7745028a0ff4e2a93427cc 2024-11-25T12:05:53,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/f8e3c2095e794259b5174ef74a7507c4 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/f8e3c2095e794259b5174ef74a7507c4 2024-11-25T12:05:53,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/d2fc910b93de4b3395cb6a52fda5e5f5 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/d2fc910b93de4b3395cb6a52fda5e5f5 2024-11-25T12:05:53,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/7cd409a13c8c4a3ca39b9b11b522e965 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/7cd409a13c8c4a3ca39b9b11b522e965 2024-11-25T12:05:53,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/5d992bb59c194aadb35e3f9095ffc435 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/5d992bb59c194aadb35e3f9095ffc435 2024-11-25T12:05:53,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/65612db69e6b41ef883e7e34c64f228f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/65612db69e6b41ef883e7e34c64f228f 2024-11-25T12:05:53,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/3b01ff2013e245e886ea991e5c68535b to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/3b01ff2013e245e886ea991e5c68535b 2024-11-25T12:05:53,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/83feecf28d6d4aaaa0cdbe4e5e1c4f2d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/83feecf28d6d4aaaa0cdbe4e5e1c4f2d 2024-11-25T12:05:53,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/4e54597d5b4d45c88e2a8790607e6988, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/3f384a9b16724546bbc9009c73f56859, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/b40c6f55603a43dfa079b9eb92149a9e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/23c3c5884c0041aab9e4710d109621eb, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/413497266ed14706b477f75b5294de8f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/dede5c64a5f94a2bb9ad490d3b373786, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/d12d148255bf4a4a97d8032508d5420f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/45969a05986a464ba2244cf40ae1ad16, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/4c316c9b3bd7408db0b53bfaaad155f2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/c49c9a1b40f94bcf823ead26934f1cd0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/289c3191d6414477a588995ee70dbbad, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/1222c39929734542a66ac9b86e9cea83, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/97dda60b1ae74e9c9d1e9881efcddb99, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/a76bcdecdb5e4d05a68640ba58eeeb5e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/04e023a95f4547d1b9848031dcdc672b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/49b954ea2f5642b891c66ddefe38b2f3, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/180da76e3e4f4daa8c5afdc7b8db7399, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/b407ab934d5f4ee78fbd432eaa5ab8c1, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/749f1dc3ce264469830e1df7147c2f9d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/73e43962bcf24bbfbac13363ab828001] to archive 2024-11-25T12:05:53,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:05:53,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/4e54597d5b4d45c88e2a8790607e6988 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/4e54597d5b4d45c88e2a8790607e6988 2024-11-25T12:05:53,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/3f384a9b16724546bbc9009c73f56859 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/3f384a9b16724546bbc9009c73f56859 2024-11-25T12:05:53,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/b40c6f55603a43dfa079b9eb92149a9e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/b40c6f55603a43dfa079b9eb92149a9e 2024-11-25T12:05:53,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/23c3c5884c0041aab9e4710d109621eb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/23c3c5884c0041aab9e4710d109621eb 2024-11-25T12:05:53,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/413497266ed14706b477f75b5294de8f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/413497266ed14706b477f75b5294de8f 2024-11-25T12:05:53,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/dede5c64a5f94a2bb9ad490d3b373786 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/dede5c64a5f94a2bb9ad490d3b373786 2024-11-25T12:05:53,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/d12d148255bf4a4a97d8032508d5420f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/d12d148255bf4a4a97d8032508d5420f 2024-11-25T12:05:53,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/45969a05986a464ba2244cf40ae1ad16 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/45969a05986a464ba2244cf40ae1ad16 2024-11-25T12:05:53,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/4c316c9b3bd7408db0b53bfaaad155f2 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/4c316c9b3bd7408db0b53bfaaad155f2 2024-11-25T12:05:53,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/c49c9a1b40f94bcf823ead26934f1cd0 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/c49c9a1b40f94bcf823ead26934f1cd0 2024-11-25T12:05:53,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/289c3191d6414477a588995ee70dbbad to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/289c3191d6414477a588995ee70dbbad 2024-11-25T12:05:53,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/1222c39929734542a66ac9b86e9cea83 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/1222c39929734542a66ac9b86e9cea83 2024-11-25T12:05:53,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/97dda60b1ae74e9c9d1e9881efcddb99 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/97dda60b1ae74e9c9d1e9881efcddb99 2024-11-25T12:05:53,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/a76bcdecdb5e4d05a68640ba58eeeb5e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/a76bcdecdb5e4d05a68640ba58eeeb5e 2024-11-25T12:05:53,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/04e023a95f4547d1b9848031dcdc672b to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/04e023a95f4547d1b9848031dcdc672b 2024-11-25T12:05:53,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/49b954ea2f5642b891c66ddefe38b2f3 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/49b954ea2f5642b891c66ddefe38b2f3 2024-11-25T12:05:53,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/180da76e3e4f4daa8c5afdc7b8db7399 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/180da76e3e4f4daa8c5afdc7b8db7399 2024-11-25T12:05:53,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/b407ab934d5f4ee78fbd432eaa5ab8c1 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/b407ab934d5f4ee78fbd432eaa5ab8c1 2024-11-25T12:05:53,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/749f1dc3ce264469830e1df7147c2f9d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/749f1dc3ce264469830e1df7147c2f9d 2024-11-25T12:05:53,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/73e43962bcf24bbfbac13363ab828001 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/73e43962bcf24bbfbac13363ab828001 2024-11-25T12:05:53,596 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/recovered.edits/416.seqid, newMaxSeqId=416, maxSeqId=1 2024-11-25T12:05:53,597 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. 2024-11-25T12:05:53,597 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1676): Region close journal for d2665826b05855fc4d139a82044b59e6: Waiting for close lock at 1732536352213Running coprocessor pre-close hooks at 1732536352213Disabling compacts and flushes for region at 1732536352213Disabling writes for close at 1732536352213Obtaining lock to block concurrent updates at 1732536352213Preparing flush snapshotting stores in d2665826b05855fc4d139a82044b59e6 at 1732536352213Finished memstore snapshotting TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732536352214 (+1 ms)Flushing stores of TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6. at 1732536352214Flushing d2665826b05855fc4d139a82044b59e6/A: creating writer at 1732536352214Flushing d2665826b05855fc4d139a82044b59e6/A: appending metadata at 1732536352218 (+4 ms)Flushing d2665826b05855fc4d139a82044b59e6/A: closing flushed file at 1732536352218Flushing d2665826b05855fc4d139a82044b59e6/B: creating writer at 1732536352627 (+409 ms)Flushing d2665826b05855fc4d139a82044b59e6/B: appending metadata at 1732536352630 (+3 ms)Flushing d2665826b05855fc4d139a82044b59e6/B: closing flushed file at 1732536352630Flushing d2665826b05855fc4d139a82044b59e6/C: creating writer at 1732536353038 (+408 ms)Flushing d2665826b05855fc4d139a82044b59e6/C: appending metadata at 1732536353040 (+2 ms)Flushing d2665826b05855fc4d139a82044b59e6/C: closing flushed file at 1732536353040Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f9d3e2b: reopening flushed file at 1732536353448 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50f25eb8: reopening flushed file at 1732536353452 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f988cf2: reopening flushed file at 1732536353456 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for d2665826b05855fc4d139a82044b59e6 in 1248ms, sequenceid=413, compaction requested=true at 1732536353461 (+5 ms)Writing region close event to WAL at 1732536353592 (+131 ms)Running coprocessor post-close hooks at 1732536353597 (+5 ms)Closed at 1732536353597 2024-11-25T12:05:53,599 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(157): Closed d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:53,600 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=d2665826b05855fc4d139a82044b59e6, regionState=CLOSED 2024-11-25T12:05:53,605 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 because future has completed 2024-11-25T12:05:53,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-25T12:05:53,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseRegionProcedure d2665826b05855fc4d139a82044b59e6, server=2bf237fd86ca,37643,1732536214674 in 1.5480 sec 2024-11-25T12:05:53,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-11-25T12:05:53,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2665826b05855fc4d139a82044b59e6, UNASSIGN in 1.5570 sec 2024-11-25T12:05:53,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-25T12:05:53,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5610 sec 2024-11-25T12:05:53,620 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536353619"}]},"ts":"1732536353619"} 2024-11-25T12:05:53,621 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T12:05:53,621 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T12:05:53,623 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5730 sec 2024-11-25T12:05:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-25T12:05:54,197 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:54,197 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-25T12:05:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:54,200 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-25T12:05:54,201 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=133, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:54,217 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:54,222 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/recovered.edits] 2024-11-25T12:05:54,225 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/1c7d751f90ca42cc870b0501f593083a to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/1c7d751f90ca42cc870b0501f593083a 2024-11-25T12:05:54,226 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/42279b2fe06446c19a5bab4d7eff8540 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/42279b2fe06446c19a5bab4d7eff8540 2024-11-25T12:05:54,227 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/6576c543b0d1473ca5d1f18eeadf3d1b to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/6576c543b0d1473ca5d1f18eeadf3d1b 2024-11-25T12:05:54,228 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/cd5e646805a44c9fb0a5ca6ec92f2104 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/A/cd5e646805a44c9fb0a5ca6ec92f2104 2024-11-25T12:05:54,230 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/16611b9f4a0b4284afeb7a5a82431b88 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/16611b9f4a0b4284afeb7a5a82431b88 2024-11-25T12:05:54,231 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/4949ef1a5281481a81e648413e130630 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/4949ef1a5281481a81e648413e130630 2024-11-25T12:05:54,233 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/b68ce32585034767b7dd7f51302a4bd4 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/b68ce32585034767b7dd7f51302a4bd4 2024-11-25T12:05:54,234 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/ffef066ad09c43199d038e1f3c7aca6f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/B/ffef066ad09c43199d038e1f3c7aca6f 2024-11-25T12:05:54,236 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/2e6eb9093845425c8246b9d6dfe08437 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/2e6eb9093845425c8246b9d6dfe08437 2024-11-25T12:05:54,237 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/59cda2bc502c44a7a9f8322655d879d6 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/59cda2bc502c44a7a9f8322655d879d6 2024-11-25T12:05:54,238 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/a268e813d240429db0ed956255c0c27e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/a268e813d240429db0ed956255c0c27e 2024-11-25T12:05:54,239 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/f8970177ebe440049d72f4387e2080f2 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/C/f8970177ebe440049d72f4387e2080f2 2024-11-25T12:05:54,243 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/recovered.edits/416.seqid to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6/recovered.edits/416.seqid 2024-11-25T12:05:54,244 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/d2665826b05855fc4d139a82044b59e6 2024-11-25T12:05:54,244 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T12:05:54,249 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=133, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:54,251 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T12:05:54,254 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T12:05:54,255 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=133, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:54,255 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T12:05:54,255 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732536354255"}]},"ts":"9223372036854775807"} 2024-11-25T12:05:54,256 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T12:05:54,258 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T12:05:54,258 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d2665826b05855fc4d139a82044b59e6, NAME => 'TestAcidGuarantees,,1732536330283.d2665826b05855fc4d139a82044b59e6.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T12:05:54,258 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T12:05:54,258 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732536354258"}]},"ts":"9223372036854775807"} 2024-11-25T12:05:54,260 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T12:05:54,261 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=133, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:54,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 64 msec 2024-11-25T12:05:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-25T12:05:54,307 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T12:05:54,307 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:54,317 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242 (was 242), OpenFileDescriptor=457 (was 465), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1363 (was 1437), ProcessCount=11 (was 11), AvailableMemoryMB=5752 (was 6643) 2024-11-25T12:05:54,326 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=242, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=1363, ProcessCount=11, AvailableMemoryMB=5751 2024-11-25T12:05:54,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T12:05:54,327 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:05:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:54,329 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T12:05:54,330 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:54,330 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 134 2024-11-25T12:05:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-25T12:05:54,331 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T12:05:54,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742349_1525 (size=959) 2024-11-25T12:05:54,337 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d 2024-11-25T12:05:54,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742350_1526 (size=53) 2024-11-25T12:05:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-25T12:05:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-25T12:05:54,743 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:05:54,743 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 23c89a604d5eddf398c6d709d0d8bf0d, disabling compactions & flushes 2024-11-25T12:05:54,743 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:54,743 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:54,743 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. after waiting 0 ms 2024-11-25T12:05:54,743 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:54,743 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:54,744 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 23c89a604d5eddf398c6d709d0d8bf0d: Waiting for close lock at 1732536354743Disabling compacts and flushes for region at 1732536354743Disabling writes for close at 1732536354743Writing region close event to WAL at 1732536354743Closed at 1732536354743 2024-11-25T12:05:54,745 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T12:05:54,745 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732536354745"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732536354745"}]},"ts":"1732536354745"} 2024-11-25T12:05:54,747 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T12:05:54,748 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T12:05:54,748 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536354748"}]},"ts":"1732536354748"} 2024-11-25T12:05:54,749 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-25T12:05:54,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23c89a604d5eddf398c6d709d0d8bf0d, ASSIGN}] 2024-11-25T12:05:54,751 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23c89a604d5eddf398c6d709d0d8bf0d, ASSIGN 2024-11-25T12:05:54,752 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=23c89a604d5eddf398c6d709d0d8bf0d, ASSIGN; state=OFFLINE, location=2bf237fd86ca,37643,1732536214674; forceNewPlan=false, retain=false 2024-11-25T12:05:54,903 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=23c89a604d5eddf398c6d709d0d8bf0d, regionState=OPENING, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:54,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=23c89a604d5eddf398c6d709d0d8bf0d, ASSIGN because future has completed 2024-11-25T12:05:54,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674}] 2024-11-25T12:05:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-25T12:05:55,060 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:55,061 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7752): Opening region: {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:05:55,061 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,061 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:05:55,061 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7794): checking encryption for 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,061 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7797): checking classloading for 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,062 INFO [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,064 INFO [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23c89a604d5eddf398c6d709d0d8bf0d columnFamilyName A 2024-11-25T12:05:55,064 DEBUG [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:55,064 INFO [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] regionserver.HStore(327): Store=23c89a604d5eddf398c6d709d0d8bf0d/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:05:55,064 INFO [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,069 INFO [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23c89a604d5eddf398c6d709d0d8bf0d columnFamilyName B 2024-11-25T12:05:55,069 DEBUG [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:55,070 INFO [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] regionserver.HStore(327): Store=23c89a604d5eddf398c6d709d0d8bf0d/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:05:55,070 INFO [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,072 INFO [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23c89a604d5eddf398c6d709d0d8bf0d columnFamilyName C 2024-11-25T12:05:55,072 DEBUG [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:55,073 INFO [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] regionserver.HStore(327): Store=23c89a604d5eddf398c6d709d0d8bf0d/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:05:55,073 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1038): replaying wal for 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,073 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,074 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,075 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1048): stopping wal replay for 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,075 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1060): Cleaning up temporary data for 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,076 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:05:55,077 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1093): writing seq id for 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,079 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T12:05:55,080 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1114): Opened 23c89a604d5eddf398c6d709d0d8bf0d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66754188, jitterRate=-0.005285084247589111}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:05:55,080 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,081 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1006): Region open journal for 23c89a604d5eddf398c6d709d0d8bf0d: Running coprocessor pre-open hook at 1732536355061Writing region info on filesystem at 1732536355061Initializing all the Stores at 1732536355062 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536355062Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536355062Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536355062Cleaning up temporary data from old regions at 1732536355075 (+13 ms)Running coprocessor post-open hooks at 1732536355080 (+5 ms)Region opened successfully at 1732536355081 (+1 ms) 2024-11-25T12:05:55,082 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., pid=136, masterSystemTime=1732536355057 2024-11-25T12:05:55,084 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:55,084 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:55,084 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=23c89a604d5eddf398c6d709d0d8bf0d, regionState=OPEN, openSeqNum=2, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:55,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 because future has completed 2024-11-25T12:05:55,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-25T12:05:55,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; OpenRegionProcedure 23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 in 181 msec 2024-11-25T12:05:55,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-25T12:05:55,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23c89a604d5eddf398c6d709d0d8bf0d, ASSIGN in 338 msec 2024-11-25T12:05:55,090 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T12:05:55,090 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536355090"}]},"ts":"1732536355090"} 2024-11-25T12:05:55,092 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-25T12:05:55,092 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T12:05:55,093 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 764 msec 2024-11-25T12:05:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-25T12:05:55,466 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:55,467 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6302c151, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:55,468 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:55,468 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:55,469 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3769dd7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:55,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:55,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:55,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:55,471 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:55,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58506359, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:55,473 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:55,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:55,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:55,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:55,476 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T12:05:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 2bf237fd86ca,40019,1732536213609 2024-11-25T12:05:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2c1712b 2024-11-25T12:05:55,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T12:05:55,478 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50914, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T12:05:55,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-25T12:05:55,480 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T12:05:55,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-25T12:05:55,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742351_1527 (size=995) 2024-11-25T12:05:55,501 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-25T12:05:55,501 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-25T12:05:55,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T12:05:55,507 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23c89a604d5eddf398c6d709d0d8bf0d, REOPEN/MOVE}] 2024-11-25T12:05:55,509 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23c89a604d5eddf398c6d709d0d8bf0d, REOPEN/MOVE 2024-11-25T12:05:55,509 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=23c89a604d5eddf398c6d709d0d8bf0d, regionState=CLOSING, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:55,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=23c89a604d5eddf398c6d709d0d8bf0d, REOPEN/MOVE because future has completed 2024-11-25T12:05:55,511 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T12:05:55,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674}] 2024-11-25T12:05:55,664 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,664 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T12:05:55,664 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 23c89a604d5eddf398c6d709d0d8bf0d, disabling compactions & flushes 2024-11-25T12:05:55,664 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:55,664 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:55,664 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. after waiting 0 ms 2024-11-25T12:05:55,664 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:55,668 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-25T12:05:55,669 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:55,669 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 23c89a604d5eddf398c6d709d0d8bf0d: Waiting for close lock at 1732536355664Running coprocessor pre-close hooks at 1732536355664Disabling compacts and flushes for region at 1732536355664Disabling writes for close at 1732536355664Writing region close event to WAL at 1732536355665 (+1 ms)Running coprocessor post-close hooks at 1732536355669 (+4 ms)Closed at 1732536355669 2024-11-25T12:05:55,669 WARN [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionServer(3299): Not adding moved region record: 23c89a604d5eddf398c6d709d0d8bf0d to self. 2024-11-25T12:05:55,671 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,672 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=23c89a604d5eddf398c6d709d0d8bf0d, regionState=CLOSED 2024-11-25T12:05:55,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 because future has completed 2024-11-25T12:05:55,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-25T12:05:55,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 in 163 msec 2024-11-25T12:05:55,677 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=23c89a604d5eddf398c6d709d0d8bf0d, REOPEN/MOVE; state=CLOSED, location=2bf237fd86ca,37643,1732536214674; forceNewPlan=false, retain=true 2024-11-25T12:05:55,828 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=23c89a604d5eddf398c6d709d0d8bf0d, regionState=OPENING, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:55,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=23c89a604d5eddf398c6d709d0d8bf0d, REOPEN/MOVE because future has completed 2024-11-25T12:05:55,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674}] 2024-11-25T12:05:55,985 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:55,985 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7752): Opening region: {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} 2024-11-25T12:05:55,985 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,986 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T12:05:55,986 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7794): checking encryption for 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,986 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7797): checking classloading for 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,987 INFO [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,987 INFO [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23c89a604d5eddf398c6d709d0d8bf0d columnFamilyName A 2024-11-25T12:05:55,988 DEBUG [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:55,991 INFO [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] regionserver.HStore(327): Store=23c89a604d5eddf398c6d709d0d8bf0d/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:05:55,991 INFO [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,993 INFO [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23c89a604d5eddf398c6d709d0d8bf0d columnFamilyName B 2024-11-25T12:05:55,993 DEBUG [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:55,993 INFO [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] regionserver.HStore(327): Store=23c89a604d5eddf398c6d709d0d8bf0d/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:05:55,993 INFO [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,994 INFO [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23c89a604d5eddf398c6d709d0d8bf0d columnFamilyName C 2024-11-25T12:05:55,994 DEBUG [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:55,994 INFO [StoreOpener-23c89a604d5eddf398c6d709d0d8bf0d-1 {}] regionserver.HStore(327): Store=23c89a604d5eddf398c6d709d0d8bf0d/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T12:05:55,994 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1038): replaying wal for 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,995 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,995 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,996 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1048): stopping wal replay for 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,996 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1060): Cleaning up temporary data for 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,996 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T12:05:55,997 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1093): writing seq id for 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,998 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1114): Opened 23c89a604d5eddf398c6d709d0d8bf0d; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68994940, jitterRate=0.028104722499847412}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T12:05:55,998 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:55,998 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1006): Region open journal for 23c89a604d5eddf398c6d709d0d8bf0d: Running coprocessor pre-open hook at 1732536355986Writing region info on filesystem at 1732536355986Initializing all the Stores at 1732536355986Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536355986Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536355987 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732536355987Cleaning up temporary data from old regions at 1732536355996 (+9 ms)Running coprocessor post-open hooks at 1732536355998 (+2 ms)Region opened successfully at 1732536355998 2024-11-25T12:05:55,999 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., pid=141, masterSystemTime=1732536355982 2024-11-25T12:05:56,001 DEBUG [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:56,001 INFO [RS_OPEN_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:56,001 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=23c89a604d5eddf398c6d709d0d8bf0d, regionState=OPEN, openSeqNum=5, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 because future has completed 2024-11-25T12:05:56,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=139 2024-11-25T12:05:56,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 in 173 msec 2024-11-25T12:05:56,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-25T12:05:56,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23c89a604d5eddf398c6d709d0d8bf0d, REOPEN/MOVE in 497 msec 2024-11-25T12:05:56,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-25T12:05:56,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 502 msec 2024-11-25T12:05:56,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 527 msec 2024-11-25T12:05:56,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-25T12:05:56,010 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:05:56,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fec01c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,011 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:56,011 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:56,012 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:56,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@571c8096, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:56,014 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:56,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d2bd54d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e177505, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,015 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:56,015 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:56,015 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:56,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:56,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:56,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b423acf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:56,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:56,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:56,017 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:56,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@232fdd5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c494dd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:56,018 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:56,018 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:56,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:56,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:56,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2711b57d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:56,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:56,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:56,020 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:56,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b2421f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,021 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a35b97a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,021 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:56,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:56,022 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:56,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:56,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:56,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b2a4f54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:56,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:56,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:56,023 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:56,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79c0989b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a0899da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,024 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:56,024 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:56,025 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ef6bd02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:56,026 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30e0628d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@325cce9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,027 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:56,027 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:56,027 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b1f42a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:56,030 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:56,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18fe693b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763f45be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,031 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:56,031 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:56,031 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eba1f48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:56,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:56,033 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a87cc79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d0c0fb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:56,037 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:56,037 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e8cd20f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:56,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:56,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:56,038 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51072, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:56,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f10529e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@461194fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,039 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:56,040 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:56,040 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:56,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:56,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:56,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e172992, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:56,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:56,041 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:56,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75c2abdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73d090d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,043 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2bf237fd86ca,40019,-1 for getting cluster id 2024-11-25T12:05:56,043 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T12:05:56,043 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '99ca1eae-ba4d-457b-be97-75f5434d5de3' 2024-11-25T12:05:56,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T12:05:56,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "99ca1eae-ba4d-457b-be97-75f5434d5de3" 2024-11-25T12:05:56,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af86afc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2bf237fd86ca,40019,-1] 2024-11-25T12:05:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T12:05:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:05:56,044 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T12:05:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a91deb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T12:05:56,045 DEBUG [Thread-2351 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:56,046 DEBUG [Thread-2353 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:56,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:56,046 DEBUG [Thread-2359 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:56,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:56,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:05:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:56,046 DEBUG [Thread-2357 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:56,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:56,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:56,049 DEBUG [Thread-2355 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:56,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:05:56,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:56,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60042, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:56,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:56,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-25T12:05:56,051 DEBUG [Thread-2362 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:56,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:05:56,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:05:56,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T12:05:56,053 DEBUG [Thread-2366 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:05:56,055 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:05:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:56,056 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:56,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:05:56,056 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:05:56,057 DEBUG [Thread-2364 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:56,057 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:05:56,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:56,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:56,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:56,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:05:56,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:56,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:05:56,061 DEBUG [Thread-2368 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:56,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:56,064 DEBUG [Thread-2370 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T12:05:56,064 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:56,065 DEBUG [Thread-2370 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2bf237fd86ca,37643,1732536214674, seqNum=-1] 2024-11-25T12:05:56,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:05:56,065 DEBUG [Thread-2370 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T12:05:56,066 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:56,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T12:05:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:05:56,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T12:05:56,069 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5] 2024-11-25T12:05:56,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d48862062f624786a11f6820339952e8_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536356060/Put/seqid=0 2024-11-25T12:05:56,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536416100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536416102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536416103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536416104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536416105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742352_1528 (size=12154) 2024-11-25T12:05:56,121 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:56,127 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d48862062f624786a11f6820339952e8_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d48862062f624786a11f6820339952e8_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:56,128 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/4aeab5bec96b4b16b951291861b885cc, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:05:56,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/4aeab5bec96b4b16b951291861b885cc is 175, key is test_row_0/A:col10/1732536356060/Put/seqid=0 2024-11-25T12:05:56,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T12:05:56,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742353_1529 (size=30955) 2024-11-25T12:05:56,159 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/4aeab5bec96b4b16b951291861b885cc 2024-11-25T12:05:56,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/ee5ca300044d485cbf1e08c01283fd80 is 50, key is test_row_0/B:col10/1732536356060/Put/seqid=0 2024-11-25T12:05:56,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T12:05:56,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:56,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:05:56,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:56,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:56,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:56,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742354_1530 (size=12001) 2024-11-25T12:05:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:56,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/ee5ca300044d485cbf1e08c01283fd80 2024-11-25T12:05:56,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536416216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536416216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536416216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536416221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536416221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/7fd22b535c984271a14ce875f6ba3459 is 50, key is test_row_0/C:col10/1732536356060/Put/seqid=0 2024-11-25T12:05:56,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742355_1531 (size=12001) 2024-11-25T12:05:56,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/7fd22b535c984271a14ce875f6ba3459 2024-11-25T12:05:56,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/4aeab5bec96b4b16b951291861b885cc as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4aeab5bec96b4b16b951291861b885cc 2024-11-25T12:05:56,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4aeab5bec96b4b16b951291861b885cc, entries=150, sequenceid=17, filesize=30.2 K 2024-11-25T12:05:56,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/ee5ca300044d485cbf1e08c01283fd80 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/ee5ca300044d485cbf1e08c01283fd80 2024-11-25T12:05:56,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/ee5ca300044d485cbf1e08c01283fd80, entries=150, sequenceid=17, filesize=11.7 K 2024-11-25T12:05:56,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/7fd22b535c984271a14ce875f6ba3459 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/7fd22b535c984271a14ce875f6ba3459 2024-11-25T12:05:56,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T12:05:56,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/7fd22b535c984271a14ce875f6ba3459, entries=150, sequenceid=17, filesize=11.7 K 2024-11-25T12:05:56,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T12:05:56,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:56,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:05:56,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:56,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 23c89a604d5eddf398c6d709d0d8bf0d in 310ms, sequenceid=17, compaction requested=false 2024-11-25T12:05:56,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:56,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:56,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:05:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:56,431 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:56,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-25T12:05:56,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536416453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536416465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252fd85e0131a542aab2d37a216b1f8c32_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536356100/Put/seqid=0 2024-11-25T12:05:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536416468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536416470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536416473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742356_1532 (size=14594) 2024-11-25T12:05:56,517 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:56,526 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252fd85e0131a542aab2d37a216b1f8c32_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252fd85e0131a542aab2d37a216b1f8c32_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:56,530 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/aa6d139ab7434d4083cc6befdd0554d5, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:05:56,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/aa6d139ab7434d4083cc6befdd0554d5 is 175, key is test_row_0/A:col10/1732536356100/Put/seqid=0 2024-11-25T12:05:56,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T12:05:56,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:56,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:05:56,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:56,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:56,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:56,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:56,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742357_1533 (size=39549) 2024-11-25T12:05:56,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536416576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536416589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536416589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536416589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536416590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T12:05:56,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T12:05:56,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:56,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:05:56,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:56,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:56,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:56,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:56,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536416786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536416795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536416796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536416796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536416796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:56,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T12:05:56,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:56,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:05:56,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:56,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:56,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:56,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:56,965 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/aa6d139ab7434d4083cc6befdd0554d5 2024-11-25T12:05:57,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/985d5b75b8a541b6a50cff9a5c9389a6 is 50, key is test_row_0/B:col10/1732536356100/Put/seqid=0 2024-11-25T12:05:57,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T12:05:57,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:57,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:05:57,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:57,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:57,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:57,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742358_1534 (size=12001) 2024-11-25T12:05:57,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/985d5b75b8a541b6a50cff9a5c9389a6 2024-11-25T12:05:57,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/d901b90d9e944c3597623c0e82eed247 is 50, key is test_row_0/C:col10/1732536356100/Put/seqid=0 2024-11-25T12:05:57,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536417102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:57,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:57,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:57,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:57,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536417106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:57,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:57,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:57,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:57,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536417106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:57,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:57,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:57,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536417106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:57,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:57,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:57,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:57,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:57,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536417116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:57,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:57,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742359_1535 (size=12001) 2024-11-25T12:05:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T12:05:57,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T12:05:57,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:57,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:05:57,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:57,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:57,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:57,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T12:05:57,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:57,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:05:57,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:57,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:57,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:57,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T12:05:57,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:57,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:05:57,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:57,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:57,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:57,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/d901b90d9e944c3597623c0e82eed247 2024-11-25T12:05:57,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/aa6d139ab7434d4083cc6befdd0554d5 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/aa6d139ab7434d4083cc6befdd0554d5 2024-11-25T12:05:57,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:57,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536417617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:57,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:57,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:57,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:57,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536417617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:57,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:57,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536417620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:57,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:57,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536417621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:57,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:57,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:57,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:57,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:57,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:57,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:57,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:57,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:57,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:57,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/aa6d139ab7434d4083cc6befdd0554d5, entries=200, sequenceid=42, filesize=38.6 K 2024-11-25T12:05:57,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536417627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:57,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:57,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:57,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:57,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/985d5b75b8a541b6a50cff9a5c9389a6 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/985d5b75b8a541b6a50cff9a5c9389a6 2024-11-25T12:05:57,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/985d5b75b8a541b6a50cff9a5c9389a6, entries=150, sequenceid=42, filesize=11.7 K 2024-11-25T12:05:57,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T12:05:57,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:57,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:05:57,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:57,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:57,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:57,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:05:57,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/d901b90d9e944c3597623c0e82eed247 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/d901b90d9e944c3597623c0e82eed247 2024-11-25T12:05:57,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/d901b90d9e944c3597623c0e82eed247, entries=150, sequenceid=42, filesize=11.7 K 2024-11-25T12:05:57,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 23c89a604d5eddf398c6d709d0d8bf0d in 1300ms, sequenceid=42, compaction requested=false 2024-11-25T12:05:57,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:05:57,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-25T12:05:57,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:57,845 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:05:57,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112545b8e3e0aa87446e90c8b23f602d668f_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536356457/Put/seqid=0 2024-11-25T12:05:57,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742360_1536 (size=12154) 2024-11-25T12:05:57,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:57,922 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112545b8e3e0aa87446e90c8b23f602d668f_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112545b8e3e0aa87446e90c8b23f602d668f_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:57,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/a714950c338243c0b3987a0b19d296df, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:05:57,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/a714950c338243c0b3987a0b19d296df is 175, key is test_row_0/A:col10/1732536356457/Put/seqid=0 2024-11-25T12:05:57,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742361_1537 (size=30955) 2024-11-25T12:05:57,957 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/a714950c338243c0b3987a0b19d296df 2024-11-25T12:05:57,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/610d9ccd39124204a531d75c687ac17d is 50, key is test_row_0/B:col10/1732536356457/Put/seqid=0 2024-11-25T12:05:58,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742362_1538 (size=12001) 2024-11-25T12:05:58,036 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/610d9ccd39124204a531d75c687ac17d 2024-11-25T12:05:58,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/81fb2d62d55a41c78bca4077ba8340fd is 50, key is test_row_0/C:col10/1732536356457/Put/seqid=0 2024-11-25T12:05:58,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742363_1539 (size=12001) 2024-11-25T12:05:58,189 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/81fb2d62d55a41c78bca4077ba8340fd 2024-11-25T12:05:58,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T12:05:58,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/a714950c338243c0b3987a0b19d296df as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/a714950c338243c0b3987a0b19d296df 2024-11-25T12:05:58,282 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/a714950c338243c0b3987a0b19d296df, entries=150, sequenceid=53, filesize=30.2 K 2024-11-25T12:05:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/610d9ccd39124204a531d75c687ac17d as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/610d9ccd39124204a531d75c687ac17d 2024-11-25T12:05:58,300 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/610d9ccd39124204a531d75c687ac17d, entries=150, sequenceid=53, filesize=11.7 K 2024-11-25T12:05:58,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/81fb2d62d55a41c78bca4077ba8340fd as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/81fb2d62d55a41c78bca4077ba8340fd 2024-11-25T12:05:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,304 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/81fb2d62d55a41c78bca4077ba8340fd, entries=150, sequenceid=53, filesize=11.7 K 2024-11-25T12:05:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,337 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 23c89a604d5eddf398c6d709d0d8bf0d in 492ms, sequenceid=53, compaction requested=true 2024-11-25T12:05:58,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:05:58,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:58,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-25T12:05:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-25T12:05:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,360 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-25T12:05:58,360 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2910 sec 2024-11-25T12:05:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 2.3210 sec 2024-11-25T12:05:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:58,722 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:05:58,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:05:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251b8df6293b8f4a4cadb1d1f26486acdb_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536358717/Put/seqid=0 2024-11-25T12:05:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742365_1541 (size=24358) 2024-11-25T12:05:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:58,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536418803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:58,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536418809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:58,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536418818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:58,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:58,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536418843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:58,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:58,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:58,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:58,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536418845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:58,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:58,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:58,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:58,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:58,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536418952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:58,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:58,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536418956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:58,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:58,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:58,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:58,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:58,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:58,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:58,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536418967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:58,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:58,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:58,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536418967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:58,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:58,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:58,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:58,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:58,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536418969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:58,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:58,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:58,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:59,142 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T12:05:59,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:59,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536419176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:59,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:59,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:59,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:59,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536419176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:59,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:59,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:59,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:59,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:59,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:59,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536419176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:59,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:59,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536419177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:59,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:59,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:59,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536419188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:59,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:59,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:59,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:59,233 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:05:59,257 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251b8df6293b8f4a4cadb1d1f26486acdb_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251b8df6293b8f4a4cadb1d1f26486acdb_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:05:59,265 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/01f872f389344c3f82744dd4bde9863f, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:05:59,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/01f872f389344c3f82744dd4bde9863f is 175, key is test_row_0/A:col10/1732536358717/Put/seqid=0 2024-11-25T12:05:59,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742364_1540 (size=73995) 2024-11-25T12:05:59,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536419487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:59,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536419487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:59,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:59,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:59,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:59,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:59,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:59,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:59,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536419492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:59,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536419498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:59,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:59,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:05:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536419501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:05:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:05:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:05:59,673 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/01f872f389344c3f82744dd4bde9863f 2024-11-25T12:05:59,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/a64a9c94df664c9a8abf25e5dbf20db2 is 50, key is test_row_0/B:col10/1732536358717/Put/seqid=0 2024-11-25T12:05:59,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742366_1542 (size=12001) 2024-11-25T12:05:59,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/a64a9c94df664c9a8abf25e5dbf20db2 2024-11-25T12:05:59,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/2916c12220184ce6a55468bf38942a29 is 50, key is test_row_0/C:col10/1732536358717/Put/seqid=0 2024-11-25T12:05:59,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742367_1543 (size=12001) 2024-11-25T12:05:59,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/2916c12220184ce6a55468bf38942a29 2024-11-25T12:05:59,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/01f872f389344c3f82744dd4bde9863f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/01f872f389344c3f82744dd4bde9863f 2024-11-25T12:05:59,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/01f872f389344c3f82744dd4bde9863f, entries=400, sequenceid=64, filesize=72.3 K 2024-11-25T12:05:59,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/a64a9c94df664c9a8abf25e5dbf20db2 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/a64a9c94df664c9a8abf25e5dbf20db2 2024-11-25T12:05:59,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/a64a9c94df664c9a8abf25e5dbf20db2, entries=150, sequenceid=64, filesize=11.7 K 2024-11-25T12:05:59,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/2916c12220184ce6a55468bf38942a29 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/2916c12220184ce6a55468bf38942a29 2024-11-25T12:05:59,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/2916c12220184ce6a55468bf38942a29, entries=150, sequenceid=64, filesize=11.7 K 2024-11-25T12:05:59,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 23c89a604d5eddf398c6d709d0d8bf0d in 1139ms, sequenceid=64, compaction requested=true 2024-11-25T12:05:59,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:05:59,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:05:59,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:05:59,862 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:59,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:05:59,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:05:59,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:05:59,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:05:59,863 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 175454 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:59,863 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/A is initiating minor compaction (all files) 2024-11-25T12:05:59,863 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/A in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:59,863 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4aeab5bec96b4b16b951291861b885cc, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/aa6d139ab7434d4083cc6befdd0554d5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/a714950c338243c0b3987a0b19d296df, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/01f872f389344c3f82744dd4bde9863f] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=171.3 K 2024-11-25T12:05:59,864 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:59,864 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4aeab5bec96b4b16b951291861b885cc, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/aa6d139ab7434d4083cc6befdd0554d5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/a714950c338243c0b3987a0b19d296df, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/01f872f389344c3f82744dd4bde9863f] 2024-11-25T12:05:59,864 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:05:59,865 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4aeab5bec96b4b16b951291861b885cc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732536356058 2024-11-25T12:05:59,865 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa6d139ab7434d4083cc6befdd0554d5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732536356100 2024-11-25T12:05:59,865 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting a714950c338243c0b3987a0b19d296df, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732536356450 2024-11-25T12:05:59,866 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01f872f389344c3f82744dd4bde9863f, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732536358678 2024-11-25T12:05:59,874 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:05:59,874 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/B is initiating minor compaction (all files) 2024-11-25T12:05:59,874 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/B in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:05:59,874 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/ee5ca300044d485cbf1e08c01283fd80, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/985d5b75b8a541b6a50cff9a5c9389a6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/610d9ccd39124204a531d75c687ac17d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/a64a9c94df664c9a8abf25e5dbf20db2] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=46.9 K 2024-11-25T12:05:59,876 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting ee5ca300044d485cbf1e08c01283fd80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732536356058 2024-11-25T12:05:59,876 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 985d5b75b8a541b6a50cff9a5c9389a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732536356100 2024-11-25T12:05:59,877 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 610d9ccd39124204a531d75c687ac17d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732536356450 2024-11-25T12:05:59,878 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting a64a9c94df664c9a8abf25e5dbf20db2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732536358710 2024-11-25T12:05:59,889 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:05:59,895 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:59,897 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/74dc19483ba041a99cfc1a483cbf1c69 is 50, key is test_row_0/B:col10/1732536358717/Put/seqid=0 2024-11-25T12:05:59,898 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411257edededcd29e442f93b8a44276eb396c_23c89a604d5eddf398c6d709d0d8bf0d store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:05:59,901 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411257edededcd29e442f93b8a44276eb396c_23c89a604d5eddf398c6d709d0d8bf0d, store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:05:59,901 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411257edededcd29e442f93b8a44276eb396c_23c89a604d5eddf398c6d709d0d8bf0d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:05:59,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742368_1544 (size=12139) 2024-11-25T12:05:59,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742369_1545 (size=4469) 2024-11-25T12:05:59,973 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#A#compaction#458 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:05:59,974 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/9d41cba9f1e84291829e89102581a3c0 is 175, key is test_row_0/A:col10/1732536358717/Put/seqid=0 2024-11-25T12:06:00,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742370_1546 (size=31093) 2024-11-25T12:06:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:00,009 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:06:00,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-25T12:06:00,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112515d3bdeef1224b5b9945a9f25e7d5cc9_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536359999/Put/seqid=0 2024-11-25T12:06:00,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742371_1547 (size=12154) 2024-11-25T12:06:00,056 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:00,063 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112515d3bdeef1224b5b9945a9f25e7d5cc9_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112515d3bdeef1224b5b9945a9f25e7d5cc9_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:00,067 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/ccb5a73801674d298e618b5e1fee0912, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:00,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/ccb5a73801674d298e618b5e1fee0912 is 175, key is test_row_0/A:col10/1732536359999/Put/seqid=0 2024-11-25T12:06:00,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536420024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:00,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536420036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:00,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536420078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:00,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536420079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:00,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536420080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:00,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742372_1548 (size=30955) 2024-11-25T12:06:00,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536420189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:00,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536420189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:00,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:00,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536420196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:00,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:00,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:00,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536420197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:00,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:00,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:00,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536420201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-25T12:06:00,232 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:06:00,237 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:06:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-25T12:06:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T12:06:00,244 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:06:00,248 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:06:00,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:06:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T12:06:00,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-25T12:06:00,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:00,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:00,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:00,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:00,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:00,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536420408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:00,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536420408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:00,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:00,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536420409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:00,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536420409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:00,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:00,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536420416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:00,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:00,425 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/74dc19483ba041a99cfc1a483cbf1c69 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/74dc19483ba041a99cfc1a483cbf1c69 2024-11-25T12:06:00,461 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/9d41cba9f1e84291829e89102581a3c0 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/9d41cba9f1e84291829e89102581a3c0 2024-11-25T12:06:00,478 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/B of 23c89a604d5eddf398c6d709d0d8bf0d into 74dc19483ba041a99cfc1a483cbf1c69(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:00,478 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:00,478 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/B, priority=12, startTime=1732536359862; duration=0sec 2024-11-25T12:06:00,479 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:06:00,479 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:B 2024-11-25T12:06:00,479 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:06:00,498 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/ccb5a73801674d298e618b5e1fee0912 2024-11-25T12:06:00,502 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:06:00,502 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/C is initiating minor compaction (all files) 2024-11-25T12:06:00,502 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/C in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:00,502 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/7fd22b535c984271a14ce875f6ba3459, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/d901b90d9e944c3597623c0e82eed247, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/81fb2d62d55a41c78bca4077ba8340fd, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/2916c12220184ce6a55468bf38942a29] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=46.9 K 2024-11-25T12:06:00,503 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fd22b535c984271a14ce875f6ba3459, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732536356058 2024-11-25T12:06:00,503 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting d901b90d9e944c3597623c0e82eed247, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732536356100 2024-11-25T12:06:00,506 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 81fb2d62d55a41c78bca4077ba8340fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732536356450 2024-11-25T12:06:00,507 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/A of 23c89a604d5eddf398c6d709d0d8bf0d into 9d41cba9f1e84291829e89102581a3c0(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:00,507 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:00,507 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/A, priority=12, startTime=1732536359862; duration=0sec 2024-11-25T12:06:00,507 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:06:00,507 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:A 2024-11-25T12:06:00,513 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 2916c12220184ce6a55468bf38942a29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732536358710 2024-11-25T12:06:00,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/0ab563e817644e3abaefe53e6bce4cec is 50, key is test_row_0/B:col10/1732536359999/Put/seqid=0 2024-11-25T12:06:00,551 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#C#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:06:00,552 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/279d745fcaae4782b5a1538b3eefc0ba is 50, key is test_row_0/C:col10/1732536358717/Put/seqid=0 2024-11-25T12:06:00,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T12:06:00,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-25T12:06:00,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:00,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:00,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:00,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:00,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:00,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742373_1549 (size=12001) 2024-11-25T12:06:00,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/0ab563e817644e3abaefe53e6bce4cec 2024-11-25T12:06:00,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/8548ff7a7d1f4c9c8b7eceae4ef12ebc is 50, key is test_row_0/C:col10/1732536359999/Put/seqid=0 2024-11-25T12:06:00,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742374_1550 (size=12139) 2024-11-25T12:06:00,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742375_1551 (size=12001) 2024-11-25T12:06:00,666 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/279d745fcaae4782b5a1538b3eefc0ba as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/279d745fcaae4782b5a1538b3eefc0ba 2024-11-25T12:06:00,675 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/C of 23c89a604d5eddf398c6d709d0d8bf0d into 279d745fcaae4782b5a1538b3eefc0ba(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:00,675 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:00,675 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/C, priority=12, startTime=1732536359862; duration=0sec 2024-11-25T12:06:00,675 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:06:00,676 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:C 2024-11-25T12:06:00,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-25T12:06:00,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:00,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:00,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:00,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:00,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:00,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536420716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:00,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:00,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536420727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:00,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536420727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:00,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536420727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:00,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536420727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:00,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:00,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T12:06:00,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-25T12:06:00,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:00,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:00,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:00,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:00,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:00,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:01,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-25T12:06:01,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:01,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:01,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:01,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:01,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:01,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:01,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/8548ff7a7d1f4c9c8b7eceae4ef12ebc 2024-11-25T12:06:01,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/ccb5a73801674d298e618b5e1fee0912 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/ccb5a73801674d298e618b5e1fee0912 2024-11-25T12:06:01,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/ccb5a73801674d298e618b5e1fee0912, entries=150, sequenceid=91, filesize=30.2 K 2024-11-25T12:06:01,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/0ab563e817644e3abaefe53e6bce4cec as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/0ab563e817644e3abaefe53e6bce4cec 2024-11-25T12:06:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/0ab563e817644e3abaefe53e6bce4cec, entries=150, sequenceid=91, filesize=11.7 K 2024-11-25T12:06:01,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/8548ff7a7d1f4c9c8b7eceae4ef12ebc as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/8548ff7a7d1f4c9c8b7eceae4ef12ebc 2024-11-25T12:06:01,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/8548ff7a7d1f4c9c8b7eceae4ef12ebc, entries=150, sequenceid=91, filesize=11.7 K 2024-11-25T12:06:01,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=53.67 KB/54960 for 23c89a604d5eddf398c6d709d0d8bf0d in 1088ms, sequenceid=91, compaction requested=false 2024-11-25T12:06:01,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-25T12:06:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:01,193 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-25T12:06:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125beb0edce1e494828ab1725e46596e3de_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_1/A:col10/1732536360022/Put/seqid=0 2024-11-25T12:06:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:01,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742376_1552 (size=9714) 2024-11-25T12:06:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:01,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536421313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536421313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:01,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536421314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:01,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536421316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:01,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536421317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:01,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T12:06:01,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536421436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:01,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536421436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:01,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536421436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536421441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:01,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536421437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:01,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:01,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:01,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536421646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:01,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536421656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:01,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536421656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:01,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536421657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:01,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536421661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:01,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:01,669 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125beb0edce1e494828ab1725e46596e3de_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125beb0edce1e494828ab1725e46596e3de_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:01,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/4acccadd6dfd4a06b60a1dacba77267f, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:01,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/4acccadd6dfd4a06b60a1dacba77267f is 175, key is test_row_1/A:col10/1732536360022/Put/seqid=0 2024-11-25T12:06:01,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742377_1553 (size=22361) 2024-11-25T12:06:01,697 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/4acccadd6dfd4a06b60a1dacba77267f 2024-11-25T12:06:01,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/ef9761a8369f4f8da9812426f38bfa49 is 50, key is test_row_1/B:col10/1732536360022/Put/seqid=0 2024-11-25T12:06:01,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742378_1554 (size=9657) 2024-11-25T12:06:01,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536421956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:01,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:01,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536421969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:01,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:01,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536421969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:01,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:01,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536421977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:01,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536421977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:01,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:01,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:01,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:02,157 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/ef9761a8369f4f8da9812426f38bfa49 2024-11-25T12:06:02,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/b30ee7424ea5476aa49ed132e821691f is 50, key is test_row_1/C:col10/1732536360022/Put/seqid=0 2024-11-25T12:06:02,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742379_1555 (size=9657) 2024-11-25T12:06:02,265 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/b30ee7424ea5476aa49ed132e821691f 2024-11-25T12:06:02,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/4acccadd6dfd4a06b60a1dacba77267f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4acccadd6dfd4a06b60a1dacba77267f 2024-11-25T12:06:02,350 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4acccadd6dfd4a06b60a1dacba77267f, entries=100, sequenceid=105, filesize=21.8 K 2024-11-25T12:06:02,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/ef9761a8369f4f8da9812426f38bfa49 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/ef9761a8369f4f8da9812426f38bfa49 2024-11-25T12:06:02,386 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/ef9761a8369f4f8da9812426f38bfa49, entries=100, sequenceid=105, filesize=9.4 K 2024-11-25T12:06:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T12:06:02,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/b30ee7424ea5476aa49ed132e821691f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/b30ee7424ea5476aa49ed132e821691f 2024-11-25T12:06:02,420 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T12:06:02,428 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/b30ee7424ea5476aa49ed132e821691f, entries=100, sequenceid=105, filesize=9.4 K 2024-11-25T12:06:02,429 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 23c89a604d5eddf398c6d709d0d8bf0d in 1237ms, sequenceid=105, compaction requested=true 2024-11-25T12:06:02,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:02,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:02,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-25T12:06:02,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-25T12:06:02,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-25T12:06:02,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1820 sec 2024-11-25T12:06:02,435 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.1960 sec 2024-11-25T12:06:02,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:02,469 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:06:02,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-25T12:06:02,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252c2823c5179d48e9b9494e57fe46959a_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536362467/Put/seqid=0 2024-11-25T12:06:02,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:02,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536422486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:02,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536422487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:02,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:02,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536422489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:02,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:02,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:02,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:02,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:02,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536422494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:02,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536422494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:02,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:02,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742380_1556 (size=12254) 2024-11-25T12:06:02,541 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:02,546 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252c2823c5179d48e9b9494e57fe46959a_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252c2823c5179d48e9b9494e57fe46959a_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:02,549 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/7b023d652d9f4560b3a680afde3d8d52, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:02,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/7b023d652d9f4560b3a680afde3d8d52 is 175, key is test_row_0/A:col10/1732536362467/Put/seqid=0 2024-11-25T12:06:02,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742381_1557 (size=31055) 2024-11-25T12:06:02,591 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/7b023d652d9f4560b3a680afde3d8d52 2024-11-25T12:06:02,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536422606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:02,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:02,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:02,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:02,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536422606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:02,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:02,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:02,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:02,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/e09e00d7a8bd4abba1717bc4d58f6fa0 is 50, key is test_row_0/B:col10/1732536362467/Put/seqid=0 2024-11-25T12:06:02,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742382_1558 (size=12101) 2024-11-25T12:06:02,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/e09e00d7a8bd4abba1717bc4d58f6fa0 2024-11-25T12:06:02,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/ee4e015b9ca44b849e7f3791c9165203 is 50, key is test_row_0/C:col10/1732536362467/Put/seqid=0 2024-11-25T12:06:02,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742383_1559 (size=12101) 2024-11-25T12:06:02,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536422817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:02,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536422817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:02,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:02,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:03,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/ee4e015b9ca44b849e7f3791c9165203 2024-11-25T12:06:03,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/7b023d652d9f4560b3a680afde3d8d52 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7b023d652d9f4560b3a680afde3d8d52 2024-11-25T12:06:03,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536423126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:03,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536423126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:03,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:03,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7b023d652d9f4560b3a680afde3d8d52, entries=150, sequenceid=132, filesize=30.3 K 2024-11-25T12:06:03,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/e09e00d7a8bd4abba1717bc4d58f6fa0 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/e09e00d7a8bd4abba1717bc4d58f6fa0 2024-11-25T12:06:03,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/e09e00d7a8bd4abba1717bc4d58f6fa0, entries=150, sequenceid=132, filesize=11.8 K 2024-11-25T12:06:03,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/ee4e015b9ca44b849e7f3791c9165203 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/ee4e015b9ca44b849e7f3791c9165203 2024-11-25T12:06:03,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/ee4e015b9ca44b849e7f3791c9165203, entries=150, sequenceid=132, filesize=11.8 K 2024-11-25T12:06:03,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 23c89a604d5eddf398c6d709d0d8bf0d in 731ms, sequenceid=132, compaction requested=true 2024-11-25T12:06:03,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:03,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:06:03,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:06:03,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:06:03,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:06:03,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:06:03,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:06:03,201 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:06:03,201 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:06:03,216 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45898 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:06:03,217 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/B is initiating minor compaction (all files) 2024-11-25T12:06:03,217 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/B in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:03,217 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/74dc19483ba041a99cfc1a483cbf1c69, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/0ab563e817644e3abaefe53e6bce4cec, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/ef9761a8369f4f8da9812426f38bfa49, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/e09e00d7a8bd4abba1717bc4d58f6fa0] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=44.8 K 2024-11-25T12:06:03,217 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 115464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:06:03,218 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74dc19483ba041a99cfc1a483cbf1c69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732536358710 2024-11-25T12:06:03,218 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/A is initiating minor compaction (all files) 2024-11-25T12:06:03,218 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/A in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:03,218 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/9d41cba9f1e84291829e89102581a3c0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/ccb5a73801674d298e618b5e1fee0912, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4acccadd6dfd4a06b60a1dacba77267f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7b023d652d9f4560b3a680afde3d8d52] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=112.8 K 2024-11-25T12:06:03,218 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:03,218 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/9d41cba9f1e84291829e89102581a3c0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/ccb5a73801674d298e618b5e1fee0912, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4acccadd6dfd4a06b60a1dacba77267f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7b023d652d9f4560b3a680afde3d8d52] 2024-11-25T12:06:03,221 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ab563e817644e3abaefe53e6bce4cec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732536358803 2024-11-25T12:06:03,221 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d41cba9f1e84291829e89102581a3c0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732536358710 2024-11-25T12:06:03,225 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting ccb5a73801674d298e618b5e1fee0912, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732536358803 2024-11-25T12:06:03,225 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef9761a8369f4f8da9812426f38bfa49, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732536360022 2024-11-25T12:06:03,226 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 4acccadd6dfd4a06b60a1dacba77267f, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732536360022 2024-11-25T12:06:03,226 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting e09e00d7a8bd4abba1717bc4d58f6fa0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536361309 2024-11-25T12:06:03,229 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b023d652d9f4560b3a680afde3d8d52, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536361309 2024-11-25T12:06:03,248 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:03,254 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#B#compaction#471 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:06:03,254 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/6c2f2e5ed27e4189a7d543c5235a9783 is 50, key is test_row_0/B:col10/1732536362467/Put/seqid=0 2024-11-25T12:06:03,259 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411252feb01dac0e1444884b62fe4b114d346_23c89a604d5eddf398c6d709d0d8bf0d store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:03,261 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411252feb01dac0e1444884b62fe4b114d346_23c89a604d5eddf398c6d709d0d8bf0d, store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:03,261 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411252feb01dac0e1444884b62fe4b114d346_23c89a604d5eddf398c6d709d0d8bf0d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:03,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742384_1560 (size=12375) 2024-11-25T12:06:03,298 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/6c2f2e5ed27e4189a7d543c5235a9783 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/6c2f2e5ed27e4189a7d543c5235a9783 2024-11-25T12:06:03,305 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/B of 23c89a604d5eddf398c6d709d0d8bf0d into 6c2f2e5ed27e4189a7d543c5235a9783(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:03,305 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:03,305 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/B, priority=12, startTime=1732536363201; duration=0sec 2024-11-25T12:06:03,305 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:06:03,305 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:B 2024-11-25T12:06:03,305 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:06:03,307 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45898 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:06:03,307 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/C is initiating minor compaction (all files) 2024-11-25T12:06:03,308 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/C in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:03,308 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/279d745fcaae4782b5a1538b3eefc0ba, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/8548ff7a7d1f4c9c8b7eceae4ef12ebc, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/b30ee7424ea5476aa49ed132e821691f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/ee4e015b9ca44b849e7f3791c9165203] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=44.8 K 2024-11-25T12:06:03,308 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 279d745fcaae4782b5a1538b3eefc0ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732536358710 2024-11-25T12:06:03,308 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8548ff7a7d1f4c9c8b7eceae4ef12ebc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732536358803 2024-11-25T12:06:03,310 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting b30ee7424ea5476aa49ed132e821691f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732536360022 2024-11-25T12:06:03,310 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee4e015b9ca44b849e7f3791c9165203, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536361309 2024-11-25T12:06:03,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742385_1561 (size=4469) 2024-11-25T12:06:03,328 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#A#compaction#470 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:06:03,328 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/c18be4f3610948088a10e58583b68e0d is 175, key is test_row_0/A:col10/1732536362467/Put/seqid=0 2024-11-25T12:06:03,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742386_1562 (size=31329) 2024-11-25T12:06:03,363 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#C#compaction#472 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:06:03,364 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/72e71991a2244c399faf00df71aeb52f is 50, key is test_row_0/C:col10/1732536362467/Put/seqid=0 2024-11-25T12:06:03,379 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/c18be4f3610948088a10e58583b68e0d as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/c18be4f3610948088a10e58583b68e0d 2024-11-25T12:06:03,384 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/A of 23c89a604d5eddf398c6d709d0d8bf0d into c18be4f3610948088a10e58583b68e0d(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:03,384 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:03,384 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/A, priority=12, startTime=1732536363201; duration=0sec 2024-11-25T12:06:03,385 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:06:03,385 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:A 2024-11-25T12:06:03,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742387_1563 (size=12375) 2024-11-25T12:06:03,419 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/72e71991a2244c399faf00df71aeb52f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/72e71991a2244c399faf00df71aeb52f 2024-11-25T12:06:03,424 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/C of 23c89a604d5eddf398c6d709d0d8bf0d into 72e71991a2244c399faf00df71aeb52f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:03,424 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:03,424 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/C, priority=12, startTime=1732536363201; duration=0sec 2024-11-25T12:06:03,424 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:06:03,424 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:C 2024-11-25T12:06:03,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:03,514 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:06:03,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T12:06:03,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112545f8ad00412a4aba81611f8c3bccd4b7_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536363508/Put/seqid=0 2024-11-25T12:06:03,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742388_1564 (size=12304) 2024-11-25T12:06:03,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536423559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:03,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:03,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536423563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:03,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536423564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:03,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536423637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:03,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:03,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536423637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:03,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:03,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536423666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:03,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:03,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:03,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536423677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:03,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536423677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:03,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536423876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:03,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536423886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:03,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:03,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536423896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:03,949 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:03,963 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112545f8ad00412a4aba81611f8c3bccd4b7_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112545f8ad00412a4aba81611f8c3bccd4b7_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:03,968 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/7e8d60930cff43dc82ce428c4946ee4e, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:03,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/7e8d60930cff43dc82ce428c4946ee4e is 175, key is test_row_0/A:col10/1732536363508/Put/seqid=0 2024-11-25T12:06:04,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742389_1565 (size=31105) 2024-11-25T12:06:04,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536424186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:04,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:04,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536424196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:04,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:04,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:04,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536424209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:04,254 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T12:06:04,254 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-25T12:06:04,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-25T12:06:04,416 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:06:04,417 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/7e8d60930cff43dc82ce428c4946ee4e 2024-11-25T12:06:04,417 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:06:04,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-25T12:06:04,420 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:06:04,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T12:06:04,421 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:06:04,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:06:04,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/d5d5dd0ee04a4e9e826268facf8e5727 is 50, key is test_row_0/B:col10/1732536363508/Put/seqid=0 2024-11-25T12:06:04,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742390_1566 (size=12151) 2024-11-25T12:06:04,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/d5d5dd0ee04a4e9e826268facf8e5727 2024-11-25T12:06:04,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/85937814111e401eaa374bfbfacc0a84 is 50, key is test_row_0/C:col10/1732536363508/Put/seqid=0 2024-11-25T12:06:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T12:06:04,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742391_1567 (size=12151) 2024-11-25T12:06:04,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/85937814111e401eaa374bfbfacc0a84 2024-11-25T12:06:04,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/7e8d60930cff43dc82ce428c4946ee4e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7e8d60930cff43dc82ce428c4946ee4e 2024-11-25T12:06:04,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T12:06:04,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:04,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:04,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:04,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:04,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:04,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:04,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7e8d60930cff43dc82ce428c4946ee4e, entries=150, sequenceid=147, filesize=30.4 K 2024-11-25T12:06:04,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/d5d5dd0ee04a4e9e826268facf8e5727 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/d5d5dd0ee04a4e9e826268facf8e5727 2024-11-25T12:06:04,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/d5d5dd0ee04a4e9e826268facf8e5727, entries=150, sequenceid=147, filesize=11.9 K 2024-11-25T12:06:04,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/85937814111e401eaa374bfbfacc0a84 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/85937814111e401eaa374bfbfacc0a84 2024-11-25T12:06:04,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/85937814111e401eaa374bfbfacc0a84, entries=150, sequenceid=147, filesize=11.9 K 2024-11-25T12:06:04,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 23c89a604d5eddf398c6d709d0d8bf0d in 1095ms, sequenceid=147, compaction requested=false 2024-11-25T12:06:04,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:04,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:04,655 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:06:04,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-25T12:06:04,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255633be70e59d4820b547ee956366c49f_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536363561/Put/seqid=0 2024-11-25T12:06:04,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536424688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:04,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:04,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:04,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:04,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536424690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:04,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:04,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:04,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:04,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:04,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536424695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:04,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:04,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742392_1568 (size=14794) 2024-11-25T12:06:04,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536424707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:04,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:04,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536424717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:04,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:04,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:04,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:04,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T12:06:04,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:04,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:04,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:04,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:04,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:04,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T12:06:04,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:04,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:04,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536424806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:04,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:04,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536424807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:04,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T12:06:04,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:04,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:04,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:04,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:04,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:04,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:05,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:05,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536425016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:05,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:05,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536425019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:05,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:05,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:05,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T12:06:05,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T12:06:05,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:05,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:05,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:05,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:05,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:05,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:05,117 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:05,128 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411255633be70e59d4820b547ee956366c49f_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255633be70e59d4820b547ee956366c49f_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:05,134 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/7f88517d91d34e46a48fc903e1221f44, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:05,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/7f88517d91d34e46a48fc903e1221f44 is 175, key is test_row_0/A:col10/1732536363561/Put/seqid=0 2024-11-25T12:06:05,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742393_1569 (size=39749) 2024-11-25T12:06:05,175 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/7f88517d91d34e46a48fc903e1221f44 2024-11-25T12:06:05,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/a4a9981e6c3a43959d1336d67546064f is 50, key is test_row_0/B:col10/1732536363561/Put/seqid=0 2024-11-25T12:06:05,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742394_1570 (size=12151) 2024-11-25T12:06:05,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/a4a9981e6c3a43959d1336d67546064f 2024-11-25T12:06:05,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T12:06:05,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:05,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:05,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:05,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:05,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:05,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/aa1168c4d10849659c16f59394d0314b is 50, key is test_row_0/C:col10/1732536363561/Put/seqid=0 2024-11-25T12:06:05,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742395_1571 (size=12151) 2024-11-25T12:06:05,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536425336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:05,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536425337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:05,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:05,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T12:06:05,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:05,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:05,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:05,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:05,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:05,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T12:06:05,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:05,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:05,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:05,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:05,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:05,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T12:06:05,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/aa1168c4d10849659c16f59394d0314b 2024-11-25T12:06:05,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T12:06:05,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:05,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:05,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:05,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:05,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:05,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536425725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:05,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536425729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:05,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:05,733 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_0' in 23c89a604d5eddf398c6d709d0d8bf0d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:06:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536425729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:05,734 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_1' in 23c89a604d5eddf398c6d709d0d8bf0d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:06:05,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:05,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:05,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:05,734 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_0' in 23c89a604d5eddf398c6d709d0d8bf0d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:06:05,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:05,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:05,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:05,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/7f88517d91d34e46a48fc903e1221f44 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7f88517d91d34e46a48fc903e1221f44 2024-11-25T12:06:05,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7f88517d91d34e46a48fc903e1221f44, entries=200, sequenceid=172, filesize=38.8 K 2024-11-25T12:06:05,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/a4a9981e6c3a43959d1336d67546064f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/a4a9981e6c3a43959d1336d67546064f 2024-11-25T12:06:05,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/a4a9981e6c3a43959d1336d67546064f, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T12:06:05,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/aa1168c4d10849659c16f59394d0314b as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/aa1168c4d10849659c16f59394d0314b 2024-11-25T12:06:05,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/aa1168c4d10849659c16f59394d0314b, entries=150, sequenceid=172, filesize=11.9 K 2024-11-25T12:06:05,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 23c89a604d5eddf398c6d709d0d8bf0d in 1190ms, sequenceid=172, compaction requested=true 2024-11-25T12:06:05,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:05,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:06:05,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:06:05,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:06:05,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:06:05,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:06:05,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:06:05,845 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:06:05,848 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:06:05,853 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:06:05,853 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/A is initiating minor compaction (all files) 2024-11-25T12:06:05,854 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/A in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:05,854 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/c18be4f3610948088a10e58583b68e0d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7e8d60930cff43dc82ce428c4946ee4e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7f88517d91d34e46a48fc903e1221f44] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=99.8 K 2024-11-25T12:06:05,854 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:05,854 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/c18be4f3610948088a10e58583b68e0d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7e8d60930cff43dc82ce428c4946ee4e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7f88517d91d34e46a48fc903e1221f44] 2024-11-25T12:06:05,854 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:06:05,854 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/C is initiating minor compaction (all files) 2024-11-25T12:06:05,854 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/C in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:05,854 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/72e71991a2244c399faf00df71aeb52f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/85937814111e401eaa374bfbfacc0a84, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/aa1168c4d10849659c16f59394d0314b] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=35.8 K 2024-11-25T12:06:05,855 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting c18be4f3610948088a10e58583b68e0d, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536361309 2024-11-25T12:06:05,856 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e8d60930cff43dc82ce428c4946ee4e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732536362479 2024-11-25T12:06:05,856 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72e71991a2244c399faf00df71aeb52f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536361309 2024-11-25T12:06:05,856 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f88517d91d34e46a48fc903e1221f44, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732536363549 2024-11-25T12:06:05,856 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85937814111e401eaa374bfbfacc0a84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732536362479 2024-11-25T12:06:05,857 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa1168c4d10849659c16f59394d0314b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732536363549 2024-11-25T12:06:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:05,861 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:06:05,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-25T12:06:05,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T12:06:05,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:05,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:05,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:05,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:05,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:05,889 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:05,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253b6a6142363d41338a146e7f41b8a0f3_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536364658/Put/seqid=0 2024-11-25T12:06:05,899 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#C#compaction#481 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:06:05,899 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/09bb1f86d31b4773a080a66ce80a5c8f is 50, key is test_row_0/C:col10/1732536363561/Put/seqid=0 2024-11-25T12:06:05,902 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411253b841a3be6814d04bf6426eeb691d8ea_23c89a604d5eddf398c6d709d0d8bf0d store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:05,903 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411253b841a3be6814d04bf6426eeb691d8ea_23c89a604d5eddf398c6d709d0d8bf0d, store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:05,903 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253b841a3be6814d04bf6426eeb691d8ea_23c89a604d5eddf398c6d709d0d8bf0d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:05,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742398_1574 (size=4469) 2024-11-25T12:06:05,945 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#A#compaction#479 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:06:05,946 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/58be2d86357d449ea0f7aa67e026d521 is 175, key is test_row_0/A:col10/1732536363561/Put/seqid=0 2024-11-25T12:06:05,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742397_1573 (size=12527) 2024-11-25T12:06:05,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742396_1572 (size=12304) 2024-11-25T12:06:05,959 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:05,975 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411253b6a6142363d41338a146e7f41b8a0f3_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253b6a6142363d41338a146e7f41b8a0f3_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:05,977 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/63d5b1f3887e4c7ca46d1c84b6b586ec, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:05,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/63d5b1f3887e4c7ca46d1c84b6b586ec is 175, key is test_row_0/A:col10/1732536364658/Put/seqid=0 2024-11-25T12:06:05,981 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/09bb1f86d31b4773a080a66ce80a5c8f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/09bb1f86d31b4773a080a66ce80a5c8f 2024-11-25T12:06:05,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742399_1575 (size=31481) 2024-11-25T12:06:06,006 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/C of 23c89a604d5eddf398c6d709d0d8bf0d into 09bb1f86d31b4773a080a66ce80a5c8f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:06,006 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:06,006 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/C, priority=13, startTime=1732536365845; duration=0sec 2024-11-25T12:06:06,006 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:06:06,006 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:C 2024-11-25T12:06:06,006 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:06:06,009 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:06:06,010 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/B is initiating minor compaction (all files) 2024-11-25T12:06:06,010 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/B in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:06,010 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/6c2f2e5ed27e4189a7d543c5235a9783, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/d5d5dd0ee04a4e9e826268facf8e5727, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/a4a9981e6c3a43959d1336d67546064f] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=35.8 K 2024-11-25T12:06:06,010 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c2f2e5ed27e4189a7d543c5235a9783, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732536361309 2024-11-25T12:06:06,011 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5d5dd0ee04a4e9e826268facf8e5727, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732536362479 2024-11-25T12:06:06,012 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4a9981e6c3a43959d1336d67546064f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732536363549 2024-11-25T12:06:06,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742400_1576 (size=31105) 2024-11-25T12:06:06,025 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=185, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/63d5b1f3887e4c7ca46d1c84b6b586ec 2024-11-25T12:06:06,026 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#B#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:06:06,027 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/39789e273399443fb45b0415a8389faa is 50, key is test_row_0/B:col10/1732536363561/Put/seqid=0 2024-11-25T12:06:06,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T12:06:06,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:06,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:06,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:06,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:06,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:06,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536426009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:06,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536426016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:06,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:06,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:06,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:06,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/39bfec65be5144c490458482cadeefcd is 50, key is test_row_0/B:col10/1732536364658/Put/seqid=0 2024-11-25T12:06:06,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742401_1577 (size=12527) 2024-11-25T12:06:06,112 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/39789e273399443fb45b0415a8389faa as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/39789e273399443fb45b0415a8389faa 2024-11-25T12:06:06,125 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/B of 23c89a604d5eddf398c6d709d0d8bf0d into 39789e273399443fb45b0415a8389faa(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:06,125 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:06,125 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/B, priority=13, startTime=1732536365845; duration=0sec 2024-11-25T12:06:06,125 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:06:06,125 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:B 2024-11-25T12:06:06,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536426137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:06,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742402_1578 (size=12151) 2024-11-25T12:06:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:06,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:06,148 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:06,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:06,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536426146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:06,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:06,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T12:06:06,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:06,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:06,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:06,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:06,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:06,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T12:06:06,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:06,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:06,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:06,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:06,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:06,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536426361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:06,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536426368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:06,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:06,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:06,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:06,417 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/58be2d86357d449ea0f7aa67e026d521 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/58be2d86357d449ea0f7aa67e026d521 2024-11-25T12:06:06,433 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/A of 23c89a604d5eddf398c6d709d0d8bf0d into 58be2d86357d449ea0f7aa67e026d521(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:06,433 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:06,433 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/A, priority=13, startTime=1732536365845; duration=0sec 2024-11-25T12:06:06,433 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:06:06,433 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:A 2024-11-25T12:06:06,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T12:06:06,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:06,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:06,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:06,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:06,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:06,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/39bfec65be5144c490458482cadeefcd 2024-11-25T12:06:06,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/5caba28645454a24aadf3fb6d815103d is 50, key is test_row_0/C:col10/1732536364658/Put/seqid=0 2024-11-25T12:06:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T12:06:06,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742403_1579 (size=12151) 2024-11-25T12:06:06,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T12:06:06,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:06,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:06,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:06,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:06,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:06,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:06,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536426677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:06,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:06,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:06,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536426690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:06,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:06,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:06,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:06,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T12:06:06,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:06,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:06,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:06,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:06,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:06,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:06,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T12:06:06,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:06,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:06,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:06,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:06,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:07,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/5caba28645454a24aadf3fb6d815103d 2024-11-25T12:06:07,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/63d5b1f3887e4c7ca46d1c84b6b586ec as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/63d5b1f3887e4c7ca46d1c84b6b586ec 2024-11-25T12:06:07,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/63d5b1f3887e4c7ca46d1c84b6b586ec, entries=150, sequenceid=185, filesize=30.4 K 2024-11-25T12:06:07,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/39bfec65be5144c490458482cadeefcd as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/39bfec65be5144c490458482cadeefcd 2024-11-25T12:06:07,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/39bfec65be5144c490458482cadeefcd, entries=150, sequenceid=185, filesize=11.9 K 2024-11-25T12:06:07,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/5caba28645454a24aadf3fb6d815103d as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/5caba28645454a24aadf3fb6d815103d 2024-11-25T12:06:07,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/5caba28645454a24aadf3fb6d815103d, entries=150, sequenceid=185, filesize=11.9 K 2024-11-25T12:06:07,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 23c89a604d5eddf398c6d709d0d8bf0d in 1196ms, sequenceid=185, compaction requested=false 2024-11-25T12:06:07,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:07,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-25T12:06:07,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:07,137 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-25T12:06:07,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d4a6f688dc23472aac15540cfc45788e_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536365966/Put/seqid=0 2024-11-25T12:06:07,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742404_1580 (size=12304) 2024-11-25T12:06:07,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:07,166 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d4a6f688dc23472aac15540cfc45788e_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d4a6f688dc23472aac15540cfc45788e_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:07,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/fe621cc5c0684866836c539f9c74cbf5, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:07,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/fe621cc5c0684866836c539f9c74cbf5 is 175, key is test_row_0/A:col10/1732536365966/Put/seqid=0 2024-11-25T12:06:07,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742405_1581 (size=31105) 2024-11-25T12:06:07,172 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/fe621cc5c0684866836c539f9c74cbf5 2024-11-25T12:06:07,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/d1cbc11325074389a0b9fcb273e316ad is 50, key is test_row_0/B:col10/1732536365966/Put/seqid=0 2024-11-25T12:06:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:07,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:07,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742406_1582 (size=12151) 2024-11-25T12:06:07,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536427245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:07,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:07,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:07,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:07,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536427250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:07,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536427356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:07,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:07,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:07,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:07,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:07,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536427369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:07,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:07,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:07,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:07,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:07,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536427568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:07,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:07,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536427589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:07,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:07,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:07,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:07,601 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/d1cbc11325074389a0b9fcb273e316ad 2024-11-25T12:06:07,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/039cd58eb8c34eb092ff18fd5100ed5f is 50, key is test_row_0/C:col10/1732536365966/Put/seqid=0 2024-11-25T12:06:07,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742407_1583 (size=12151) 2024-11-25T12:06:07,661 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/039cd58eb8c34eb092ff18fd5100ed5f 2024-11-25T12:06:07,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/fe621cc5c0684866836c539f9c74cbf5 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/fe621cc5c0684866836c539f9c74cbf5 2024-11-25T12:06:07,696 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/fe621cc5c0684866836c539f9c74cbf5, entries=150, sequenceid=211, filesize=30.4 K 2024-11-25T12:06:07,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/d1cbc11325074389a0b9fcb273e316ad as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/d1cbc11325074389a0b9fcb273e316ad 2024-11-25T12:06:07,733 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/d1cbc11325074389a0b9fcb273e316ad, entries=150, sequenceid=211, filesize=11.9 K 2024-11-25T12:06:07,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/039cd58eb8c34eb092ff18fd5100ed5f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/039cd58eb8c34eb092ff18fd5100ed5f 2024-11-25T12:06:07,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536427747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:07,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:07,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536427748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:07,751 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_0' in 23c89a604d5eddf398c6d709d0d8bf0d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:06:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:07,753 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_0' in 23c89a604d5eddf398c6d709d0d8bf0d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:06:07,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:07,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:07,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:07,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:07,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536427756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:07,760 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_1' in 23c89a604d5eddf398c6d709d0d8bf0d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:06:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:07,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:07,780 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/039cd58eb8c34eb092ff18fd5100ed5f, entries=150, sequenceid=211, filesize=11.9 K 2024-11-25T12:06:07,784 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 23c89a604d5eddf398c6d709d0d8bf0d in 647ms, sequenceid=211, compaction requested=true 2024-11-25T12:06:07,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:07,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:07,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-25T12:06:07,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-25T12:06:07,795 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-25T12:06:07,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3720 sec 2024-11-25T12:06:07,800 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 3.3820 sec 2024-11-25T12:06:07,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:07,893 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:06:07,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-25T12:06:07,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b7c26231f3a5447cb7c3a9ee7e278c36_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536367877/Put/seqid=0 2024-11-25T12:06:07,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742408_1584 (size=14794) 2024-11-25T12:06:07,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:07,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536427985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:07,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:07,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536427992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:08,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:08,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:08,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:08,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536428101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:08,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:08,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:08,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:08,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536428107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:08,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536428316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:08,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536428316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:08,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:08,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:08,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:08,360 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:08,372 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125b7c26231f3a5447cb7c3a9ee7e278c36_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b7c26231f3a5447cb7c3a9ee7e278c36_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:08,376 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/02ccf7e606434494b1fe29a73752c4f7, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:08,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/02ccf7e606434494b1fe29a73752c4f7 is 175, key is test_row_0/A:col10/1732536367877/Put/seqid=0 2024-11-25T12:06:08,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742409_1585 (size=39749) 2024-11-25T12:06:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-25T12:06:08,616 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:06:08,621 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:06:08,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-25T12:06:08,630 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:06:08,631 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:06:08,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:06:08,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T12:06:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536428626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:08,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:08,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536428626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:08,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:08,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T12:06:08,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T12:06:08,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:08,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:08,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:08,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:08,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:08,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:08,819 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/02ccf7e606434494b1fe29a73752c4f7 2024-11-25T12:06:08,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/59745b552eab44c7afc8228cb6e15e30 is 50, key is test_row_0/B:col10/1732536367877/Put/seqid=0 2024-11-25T12:06:08,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742410_1586 (size=12151) 2024-11-25T12:06:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T12:06:08,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T12:06:08,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:08,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:08,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:08,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:08,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:09,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T12:06:09,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:09,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:09,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:09,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:09,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:09,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536429146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:09,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536429146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:09,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:09,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:09,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T12:06:09,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T12:06:09,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:09,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:09,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:09,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:09,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:09,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/59745b552eab44c7afc8228cb6e15e30 2024-11-25T12:06:09,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/d326b49a5945424a978572c75765ab0c is 50, key is test_row_0/C:col10/1732536367877/Put/seqid=0 2024-11-25T12:06:09,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742411_1587 (size=12151) 2024-11-25T12:06:09,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T12:06:09,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:09,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:09,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:09,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:09,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:09,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T12:06:09,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:09,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:09,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:09,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:09,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:09,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/d326b49a5945424a978572c75765ab0c 2024-11-25T12:06:09,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/02ccf7e606434494b1fe29a73752c4f7 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/02ccf7e606434494b1fe29a73752c4f7 2024-11-25T12:06:09,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/02ccf7e606434494b1fe29a73752c4f7, entries=200, sequenceid=225, filesize=38.8 K 2024-11-25T12:06:09,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/59745b552eab44c7afc8228cb6e15e30 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/59745b552eab44c7afc8228cb6e15e30 2024-11-25T12:06:09,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/59745b552eab44c7afc8228cb6e15e30, entries=150, sequenceid=225, filesize=11.9 K 2024-11-25T12:06:09,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/d326b49a5945424a978572c75765ab0c as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/d326b49a5945424a978572c75765ab0c 2024-11-25T12:06:09,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/d326b49a5945424a978572c75765ab0c, entries=150, sequenceid=225, filesize=11.9 K 2024-11-25T12:06:09,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 23c89a604d5eddf398c6d709d0d8bf0d in 1858ms, sequenceid=225, compaction requested=true 2024-11-25T12:06:09,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:09,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:06:09,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:06:09,752 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:06:09,752 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:06:09,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:06:09,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:06:09,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:06:09,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:06:09,754 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133440 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:06:09,754 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/A is initiating minor compaction (all files) 2024-11-25T12:06:09,754 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/A in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:09,754 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/58be2d86357d449ea0f7aa67e026d521, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/63d5b1f3887e4c7ca46d1c84b6b586ec, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/fe621cc5c0684866836c539f9c74cbf5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/02ccf7e606434494b1fe29a73752c4f7] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=130.3 K 2024-11-25T12:06:09,754 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:09,754 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/58be2d86357d449ea0f7aa67e026d521, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/63d5b1f3887e4c7ca46d1c84b6b586ec, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/fe621cc5c0684866836c539f9c74cbf5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/02ccf7e606434494b1fe29a73752c4f7] 2024-11-25T12:06:09,755 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:06:09,755 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/B is initiating minor compaction (all files) 2024-11-25T12:06:09,755 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/B in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:09,755 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/39789e273399443fb45b0415a8389faa, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/39bfec65be5144c490458482cadeefcd, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/d1cbc11325074389a0b9fcb273e316ad, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/59745b552eab44c7afc8228cb6e15e30] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=47.8 K 2024-11-25T12:06:09,755 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58be2d86357d449ea0f7aa67e026d521, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732536363549 2024-11-25T12:06:09,755 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 39789e273399443fb45b0415a8389faa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732536363549 2024-11-25T12:06:09,756 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 39bfec65be5144c490458482cadeefcd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732536364658 2024-11-25T12:06:09,756 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63d5b1f3887e4c7ca46d1c84b6b586ec, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732536364658 2024-11-25T12:06:09,756 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting d1cbc11325074389a0b9fcb273e316ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732536365966 2024-11-25T12:06:09,756 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe621cc5c0684866836c539f9c74cbf5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732536365966 2024-11-25T12:06:09,757 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 59745b552eab44c7afc8228cb6e15e30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732536367229 2024-11-25T12:06:09,757 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02ccf7e606434494b1fe29a73752c4f7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732536367229 2024-11-25T12:06:09,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-25T12:06:09,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:09,763 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-25T12:06:09,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T12:06:09,773 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:09,775 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#B#compaction#491 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:06:09,775 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/98d822cd6dea46d78b1803e3704aa925 is 50, key is test_row_0/B:col10/1732536367877/Put/seqid=0 2024-11-25T12:06:09,781 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411257fe32169151a4ae0bb2ba68e5c6028e1_23c89a604d5eddf398c6d709d0d8bf0d store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:09,784 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411257fe32169151a4ae0bb2ba68e5c6028e1_23c89a604d5eddf398c6d709d0d8bf0d, store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:09,784 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411257fe32169151a4ae0bb2ba68e5c6028e1_23c89a604d5eddf398c6d709d0d8bf0d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:09,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742412_1588 (size=12663) 2024-11-25T12:06:09,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125cab45ed80fa742ff971595d9f7a6efce_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536367955/Put/seqid=0 2024-11-25T12:06:09,863 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/98d822cd6dea46d78b1803e3704aa925 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/98d822cd6dea46d78b1803e3704aa925 2024-11-25T12:06:09,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742413_1589 (size=4469) 2024-11-25T12:06:09,867 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#A#compaction#492 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:06:09,867 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/74adf05786a84d03ab402b4be57d34ba is 175, key is test_row_0/A:col10/1732536367877/Put/seqid=0 2024-11-25T12:06:09,869 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/B of 23c89a604d5eddf398c6d709d0d8bf0d into 98d822cd6dea46d78b1803e3704aa925(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:09,869 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:09,869 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/B, priority=12, startTime=1732536369752; duration=0sec 2024-11-25T12:06:09,870 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:06:09,870 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:B 2024-11-25T12:06:09,870 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:06:09,871 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:06:09,871 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/C is initiating minor compaction (all files) 2024-11-25T12:06:09,871 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/C in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:09,871 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/09bb1f86d31b4773a080a66ce80a5c8f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/5caba28645454a24aadf3fb6d815103d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/039cd58eb8c34eb092ff18fd5100ed5f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/d326b49a5945424a978572c75765ab0c] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=47.8 K 2024-11-25T12:06:09,871 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 09bb1f86d31b4773a080a66ce80a5c8f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732536363549 2024-11-25T12:06:09,872 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 5caba28645454a24aadf3fb6d815103d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732536364658 2024-11-25T12:06:09,872 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 039cd58eb8c34eb092ff18fd5100ed5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732536365966 2024-11-25T12:06:09,872 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting d326b49a5945424a978572c75765ab0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732536367229 2024-11-25T12:06:09,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742415_1591 (size=31617) 2024-11-25T12:06:09,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742414_1590 (size=12304) 2024-11-25T12:06:09,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:09,905 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125cab45ed80fa742ff971595d9f7a6efce_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cab45ed80fa742ff971595d9f7a6efce_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:09,912 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#C#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:06:09,912 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/e3c269cd715e4f2c8ef137dc04f3c544 is 50, key is test_row_0/C:col10/1732536367877/Put/seqid=0 2024-11-25T12:06:09,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/18007c66f7514043b1219c607ce651e1, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:09,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/18007c66f7514043b1219c607ce651e1 is 175, key is test_row_0/A:col10/1732536367955/Put/seqid=0 2024-11-25T12:06:09,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742416_1592 (size=12663) 2024-11-25T12:06:09,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742417_1593 (size=31105) 2024-11-25T12:06:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:10,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:10,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536430219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:10,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:10,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:10,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536430226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:10,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:10,322 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/74adf05786a84d03ab402b4be57d34ba as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/74adf05786a84d03ab402b4be57d34ba 2024-11-25T12:06:10,350 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/A of 23c89a604d5eddf398c6d709d0d8bf0d into 74adf05786a84d03ab402b4be57d34ba(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:10,350 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:10,350 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/A, priority=12, startTime=1732536369751; duration=0sec 2024-11-25T12:06:10,350 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:06:10,350 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:A 2024-11-25T12:06:10,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536430349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:10,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536430349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:10,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:10,365 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/18007c66f7514043b1219c607ce651e1 2024-11-25T12:06:10,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/900d974f804f4a0791fd7eee1b704b54 is 50, key is test_row_0/B:col10/1732536367955/Put/seqid=0 2024-11-25T12:06:10,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742418_1594 (size=12151) 2024-11-25T12:06:10,411 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/e3c269cd715e4f2c8ef137dc04f3c544 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/e3c269cd715e4f2c8ef137dc04f3c544 2024-11-25T12:06:10,417 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/900d974f804f4a0791fd7eee1b704b54 2024-11-25T12:06:10,432 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/C of 23c89a604d5eddf398c6d709d0d8bf0d into e3c269cd715e4f2c8ef137dc04f3c544(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:10,432 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:10,432 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/C, priority=12, startTime=1732536369753; duration=0sec 2024-11-25T12:06:10,434 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:06:10,434 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:C 2024-11-25T12:06:10,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/4ea0cf819cc8450c8018212b6f73756a is 50, key is test_row_0/C:col10/1732536367955/Put/seqid=0 2024-11-25T12:06:10,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742419_1595 (size=12151) 2024-11-25T12:06:10,538 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/4ea0cf819cc8450c8018212b6f73756a 2024-11-25T12:06:10,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536430566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:10,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536430566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:10,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:10,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:10,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/18007c66f7514043b1219c607ce651e1 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/18007c66f7514043b1219c607ce651e1 2024-11-25T12:06:10,661 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/18007c66f7514043b1219c607ce651e1, entries=150, sequenceid=247, filesize=30.4 K 2024-11-25T12:06:10,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/900d974f804f4a0791fd7eee1b704b54 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/900d974f804f4a0791fd7eee1b704b54 2024-11-25T12:06:10,702 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/900d974f804f4a0791fd7eee1b704b54, entries=150, sequenceid=247, filesize=11.9 K 2024-11-25T12:06:10,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/4ea0cf819cc8450c8018212b6f73756a as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/4ea0cf819cc8450c8018212b6f73756a 2024-11-25T12:06:10,716 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/4ea0cf819cc8450c8018212b6f73756a, entries=150, sequenceid=247, filesize=11.9 K 2024-11-25T12:06:10,720 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 23c89a604d5eddf398c6d709d0d8bf0d in 956ms, sequenceid=247, compaction requested=false 2024-11-25T12:06:10,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:10,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:10,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-25T12:06:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-25T12:06:10,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-25T12:06:10,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1030 sec 2024-11-25T12:06:10,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.1190 sec 2024-11-25T12:06:10,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-25T12:06:10,776 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:06:10,778 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:06:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-25T12:06:10,781 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:06:10,781 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:06:10,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:06:10,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T12:06:10,885 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:06:10,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-25T12:06:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:10,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T12:06:10,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112532f77b1a97534300aab941a4a44da106_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536370200/Put/seqid=0 2024-11-25T12:06:10,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T12:06:10,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:10,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:10,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:10,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:10,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:10,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:10,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742420_1596 (size=14994) 2024-11-25T12:06:10,940 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:10,946 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112532f77b1a97534300aab941a4a44da106_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112532f77b1a97534300aab941a4a44da106_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:10,947 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/921e5f2401034ac98733c743a1485f92, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:10,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/921e5f2401034ac98733c743a1485f92 is 175, key is test_row_0/A:col10/1732536370200/Put/seqid=0 2024-11-25T12:06:10,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742421_1597 (size=39949) 2024-11-25T12:06:10,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:10,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536430964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:10,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536430965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:11,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:11,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536431077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:11,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:11,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:11,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:11,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536431081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:11,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:11,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:11,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:11,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T12:06:11,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:11,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:11,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:11,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:11,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:11,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:11,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T12:06:11,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T12:06:11,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:11,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:11,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:11,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:11,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:11,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:11,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536431293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:11,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:11,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536431296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:11,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:11,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:11,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:11,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:11,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:11,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:11,364 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/921e5f2401034ac98733c743a1485f92 2024-11-25T12:06:11,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/5ee64e981c2345f7ad7f7742141b67d7 is 50, key is test_row_0/B:col10/1732536370200/Put/seqid=0 2024-11-25T12:06:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T12:06:11,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742422_1598 (size=12301) 2024-11-25T12:06:11,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T12:06:11,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:11,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:11,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:11,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:11,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:11,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T12:06:11,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:11,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:11,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:11,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:11,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:11,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:11,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536431606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:11,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:11,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:11,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536431609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:11,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:11,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:11,613 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:11,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T12:06:11,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:11,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:11,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:11,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:11,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:11,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:11,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60042 deadline: 1732536431769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:11,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60054 deadline: 1732536431766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:11,781 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_0' in 23c89a604d5eddf398c6d709d0d8bf0d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:06:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:11,781 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_0' in 23c89a604d5eddf398c6d709d0d8bf0d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:06:11,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:11,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:11,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:11,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60026 deadline: 1732536431787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:11,794 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2bf237fd86ca,37643,1732536214674 for 'test_row_1' in 23c89a604d5eddf398c6d709d0d8bf0d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-25T12:06:11,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:11,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:11,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:11,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/5ee64e981c2345f7ad7f7742141b67d7 2024-11-25T12:06:11,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/ef2c25b4fbd34241b642bf871d050063 is 50, key is test_row_0/C:col10/1732536370200/Put/seqid=0 2024-11-25T12:06:11,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742423_1599 (size=12301) 2024-11-25T12:06:11,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T12:06:11,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:11,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:11,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:11,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:11,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:11,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/ef2c25b4fbd34241b642bf871d050063 2024-11-25T12:06:11,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/921e5f2401034ac98733c743a1485f92 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/921e5f2401034ac98733c743a1485f92 2024-11-25T12:06:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T12:06:11,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/921e5f2401034ac98733c743a1485f92, entries=200, sequenceid=267, filesize=39.0 K 2024-11-25T12:06:11,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/5ee64e981c2345f7ad7f7742141b67d7 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/5ee64e981c2345f7ad7f7742141b67d7 2024-11-25T12:06:11,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/5ee64e981c2345f7ad7f7742141b67d7, entries=150, sequenceid=267, filesize=12.0 K 2024-11-25T12:06:11,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/ef2c25b4fbd34241b642bf871d050063 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/ef2c25b4fbd34241b642bf871d050063 2024-11-25T12:06:11,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/ef2c25b4fbd34241b642bf871d050063, entries=150, sequenceid=267, filesize=12.0 K 2024-11-25T12:06:11,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 23c89a604d5eddf398c6d709d0d8bf0d in 1086ms, sequenceid=267, compaction requested=true 2024-11-25T12:06:11,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:11,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:06:11,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:06:11,971 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:06:11,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:06:11,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:06:11,971 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:06:11,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:06:11,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:06:11,972 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102671 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:06:11,973 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:06:11,973 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/A is initiating minor compaction (all files) 2024-11-25T12:06:11,973 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/B is initiating minor compaction (all files) 2024-11-25T12:06:11,973 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/B in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:11,973 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/98d822cd6dea46d78b1803e3704aa925, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/900d974f804f4a0791fd7eee1b704b54, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/5ee64e981c2345f7ad7f7742141b67d7] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=36.2 K 2024-11-25T12:06:11,973 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/A in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:11,973 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/74adf05786a84d03ab402b4be57d34ba, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/18007c66f7514043b1219c607ce651e1, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/921e5f2401034ac98733c743a1485f92] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=100.3 K 2024-11-25T12:06:11,973 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:11,973 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/74adf05786a84d03ab402b4be57d34ba, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/18007c66f7514043b1219c607ce651e1, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/921e5f2401034ac98733c743a1485f92] 2024-11-25T12:06:11,974 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 98d822cd6dea46d78b1803e3704aa925, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732536367229 2024-11-25T12:06:11,976 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74adf05786a84d03ab402b4be57d34ba, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732536367229 2024-11-25T12:06:11,980 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 900d974f804f4a0791fd7eee1b704b54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732536367955 2024-11-25T12:06:11,981 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18007c66f7514043b1219c607ce651e1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732536367955 2024-11-25T12:06:11,981 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ee64e981c2345f7ad7f7742141b67d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732536370191 2024-11-25T12:06:11,984 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 921e5f2401034ac98733c743a1485f92, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732536370191 2024-11-25T12:06:12,026 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:12,030 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#B#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:06:12,041 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/03bff1b1f4364765a81989b38609e6e3 is 50, key is test_row_0/B:col10/1732536370200/Put/seqid=0 2024-11-25T12:06:12,045 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411251b159d3dab3a4530933f64bdc779d7ce_23c89a604d5eddf398c6d709d0d8bf0d store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:12,047 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411251b159d3dab3a4530933f64bdc779d7ce_23c89a604d5eddf398c6d709d0d8bf0d, store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:12,047 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411251b159d3dab3a4530933f64bdc779d7ce_23c89a604d5eddf398c6d709d0d8bf0d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:12,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-25T12:06:12,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:12,056 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-25T12:06:12,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742424_1600 (size=12915) 2024-11-25T12:06:12,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125db9b31b6663d4560a71d211b4e257abf_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536370959/Put/seqid=0 2024-11-25T12:06:12,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742425_1601 (size=4469) 2024-11-25T12:06:12,113 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#A#compaction#500 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:06:12,117 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/45d995ff357644ce87ac2fecdbb13f73 is 175, key is test_row_0/A:col10/1732536370200/Put/seqid=0 2024-11-25T12:06:12,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:12,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742426_1602 (size=12454) 2024-11-25T12:06:12,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:12,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742427_1603 (size=31869) 2024-11-25T12:06:12,187 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125db9b31b6663d4560a71d211b4e257abf_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125db9b31b6663d4560a71d211b4e257abf_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:12,193 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/45d995ff357644ce87ac2fecdbb13f73 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/45d995ff357644ce87ac2fecdbb13f73 2024-11-25T12:06:12,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/d5393cf63d8c48da9b8a3e3e6e45d4d6, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:12,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/d5393cf63d8c48da9b8a3e3e6e45d4d6 is 175, key is test_row_0/A:col10/1732536370959/Put/seqid=0 2024-11-25T12:06:12,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536432187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:12,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:12,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:12,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:12,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536432195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:12,233 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/A of 23c89a604d5eddf398c6d709d0d8bf0d into 45d995ff357644ce87ac2fecdbb13f73(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:12,233 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:12,234 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/A, priority=13, startTime=1732536371971; duration=0sec 2024-11-25T12:06:12,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742428_1604 (size=31255) 2024-11-25T12:06:12,234 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:06:12,234 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:A 2024-11-25T12:06:12,234 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:06:12,236 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/d5393cf63d8c48da9b8a3e3e6e45d4d6 2024-11-25T12:06:12,240 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:06:12,240 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/C is initiating minor compaction (all files) 2024-11-25T12:06:12,240 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/C in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:12,240 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/e3c269cd715e4f2c8ef137dc04f3c544, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/4ea0cf819cc8450c8018212b6f73756a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/ef2c25b4fbd34241b642bf871d050063] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=36.2 K 2024-11-25T12:06:12,241 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3c269cd715e4f2c8ef137dc04f3c544, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732536367229 2024-11-25T12:06:12,241 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ea0cf819cc8450c8018212b6f73756a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732536367955 2024-11-25T12:06:12,244 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef2c25b4fbd34241b642bf871d050063, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732536370191 2024-11-25T12:06:12,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/2b9725b34b954355aed6c4cdc559dd91 is 50, key is test_row_0/B:col10/1732536370959/Put/seqid=0 2024-11-25T12:06:12,280 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#C#compaction#504 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:06:12,280 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/edf8185fb32e4caa9284ff8ebcdbbe82 is 50, key is test_row_0/C:col10/1732536370200/Put/seqid=0 2024-11-25T12:06:12,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742429_1605 (size=12301) 2024-11-25T12:06:12,303 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/2b9725b34b954355aed6c4cdc559dd91 2024-11-25T12:06:12,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536432309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:12,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536432309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:12,313 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:12,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742430_1606 (size=12915) 2024-11-25T12:06:12,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/5eca799bdbdb49ea832fc44d1ab12720 is 50, key is test_row_0/C:col10/1732536370959/Put/seqid=0 2024-11-25T12:06:12,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742431_1607 (size=12301) 2024-11-25T12:06:12,485 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/03bff1b1f4364765a81989b38609e6e3 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/03bff1b1f4364765a81989b38609e6e3 2024-11-25T12:06:12,491 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/B of 23c89a604d5eddf398c6d709d0d8bf0d into 03bff1b1f4364765a81989b38609e6e3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:12,491 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:12,491 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/B, priority=13, startTime=1732536371971; duration=0sec 2024-11-25T12:06:12,491 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:06:12,491 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:B 2024-11-25T12:06:12,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536432516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:12,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:12,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536432517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:12,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:12,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:12,730 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/edf8185fb32e4caa9284ff8ebcdbbe82 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/edf8185fb32e4caa9284ff8ebcdbbe82 2024-11-25T12:06:12,737 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/C of 23c89a604d5eddf398c6d709d0d8bf0d into edf8185fb32e4caa9284ff8ebcdbbe82(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:12,737 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:12,737 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/C, priority=13, startTime=1732536371971; duration=0sec 2024-11-25T12:06:12,737 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:06:12,737 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:C 2024-11-25T12:06:12,751 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/5eca799bdbdb49ea832fc44d1ab12720 2024-11-25T12:06:12,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/d5393cf63d8c48da9b8a3e3e6e45d4d6 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/d5393cf63d8c48da9b8a3e3e6e45d4d6 2024-11-25T12:06:12,778 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/d5393cf63d8c48da9b8a3e3e6e45d4d6, entries=150, sequenceid=287, filesize=30.5 K 2024-11-25T12:06:12,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/2b9725b34b954355aed6c4cdc559dd91 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/2b9725b34b954355aed6c4cdc559dd91 2024-11-25T12:06:12,782 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/2b9725b34b954355aed6c4cdc559dd91, entries=150, sequenceid=287, filesize=12.0 K 2024-11-25T12:06:12,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/5eca799bdbdb49ea832fc44d1ab12720 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/5eca799bdbdb49ea832fc44d1ab12720 2024-11-25T12:06:12,786 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/5eca799bdbdb49ea832fc44d1ab12720, entries=150, sequenceid=287, filesize=12.0 K 2024-11-25T12:06:12,787 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 23c89a604d5eddf398c6d709d0d8bf0d in 731ms, sequenceid=287, compaction requested=false 2024-11-25T12:06:12,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:12,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:12,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-25T12:06:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-25T12:06:12,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-25T12:06:12,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0070 sec 2024-11-25T12:06:12,793 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.0130 sec 2024-11-25T12:06:12,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:12,830 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:06:12,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-25T12:06:12,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112515f3fed3ba9f4e4387d4afbefb8fb0ca_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536372829/Put/seqid=0 2024-11-25T12:06:12,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742432_1608 (size=17534) 2024-11-25T12:06:12,850 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:12,854 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112515f3fed3ba9f4e4387d4afbefb8fb0ca_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112515f3fed3ba9f4e4387d4afbefb8fb0ca_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:12,855 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/4971a8a46a9143b9a7c9ca8f83b7c92f, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:12,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/4971a8a46a9143b9a7c9ca8f83b7c92f is 175, key is test_row_0/A:col10/1732536372829/Put/seqid=0 2024-11-25T12:06:12,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742433_1609 (size=48639) 2024-11-25T12:06:12,863 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/4971a8a46a9143b9a7c9ca8f83b7c92f 2024-11-25T12:06:12,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/3db00a56dd32413d92d6d63e4a735377 is 50, key is test_row_0/B:col10/1732536372829/Put/seqid=0 2024-11-25T12:06:12,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742434_1610 (size=12301) 2024-11-25T12:06:12,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:12,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536432871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:12,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536432872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:12,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:12,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:12,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:12,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-25T12:06:12,937 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:06:12,937 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:06:12,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-25T12:06:12,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T12:06:12,940 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:06:12,940 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:06:12,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:06:12,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:12,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536432986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:12,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:12,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:12,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:12,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:12,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536432986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:12,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:12,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:12,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T12:06:13,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-25T12:06:13,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:13,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:13,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:13,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:13,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:13,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:13,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:13,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536433197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:13,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:13,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:13,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:13,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536433197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:13,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:13,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:13,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:13,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-25T12:06:13,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:13,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:13,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:13,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:13,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T12:06:13,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/3db00a56dd32413d92d6d63e4a735377 2024-11-25T12:06:13,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/bb465d04a92d44e58d653388f78c7c3a is 50, key is test_row_0/C:col10/1732536372829/Put/seqid=0 2024-11-25T12:06:13,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742435_1611 (size=12301) 2024-11-25T12:06:13,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/bb465d04a92d44e58d653388f78c7c3a 2024-11-25T12:06:13,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/4971a8a46a9143b9a7c9ca8f83b7c92f as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4971a8a46a9143b9a7c9ca8f83b7c92f 2024-11-25T12:06:13,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4971a8a46a9143b9a7c9ca8f83b7c92f, entries=250, sequenceid=308, filesize=47.5 K 2024-11-25T12:06:13,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/3db00a56dd32413d92d6d63e4a735377 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/3db00a56dd32413d92d6d63e4a735377 2024-11-25T12:06:13,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-25T12:06:13,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:13,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:13,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:13,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:13,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:13,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/3db00a56dd32413d92d6d63e4a735377, entries=150, sequenceid=308, filesize=12.0 K 2024-11-25T12:06:13,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/bb465d04a92d44e58d653388f78c7c3a as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/bb465d04a92d44e58d653388f78c7c3a 2024-11-25T12:06:13,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/bb465d04a92d44e58d653388f78c7c3a, entries=150, sequenceid=308, filesize=12.0 K 2024-11-25T12:06:13,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 23c89a604d5eddf398c6d709d0d8bf0d in 664ms, sequenceid=308, compaction requested=true 2024-11-25T12:06:13,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:13,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:06:13,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:06:13,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:06:13,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:06:13,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:06:13,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:06:13,496 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:06:13,496 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:06:13,512 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111763 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:06:13,512 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/A is initiating minor compaction (all files) 2024-11-25T12:06:13,513 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/A in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:13,513 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/45d995ff357644ce87ac2fecdbb13f73, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/d5393cf63d8c48da9b8a3e3e6e45d4d6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4971a8a46a9143b9a7c9ca8f83b7c92f] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=109.1 K 2024-11-25T12:06:13,513 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:13,513 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/45d995ff357644ce87ac2fecdbb13f73, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/d5393cf63d8c48da9b8a3e3e6e45d4d6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4971a8a46a9143b9a7c9ca8f83b7c92f] 2024-11-25T12:06:13,513 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:06:13,513 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/C is initiating minor compaction (all files) 2024-11-25T12:06:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:13,513 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/C in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:13,513 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:06:13,513 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/edf8185fb32e4caa9284ff8ebcdbbe82, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/5eca799bdbdb49ea832fc44d1ab12720, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/bb465d04a92d44e58d653388f78c7c3a] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=36.6 K 2024-11-25T12:06:13,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-25T12:06:13,514 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting edf8185fb32e4caa9284ff8ebcdbbe82, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732536370191 2024-11-25T12:06:13,514 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45d995ff357644ce87ac2fecdbb13f73, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732536370191 2024-11-25T12:06:13,514 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 5eca799bdbdb49ea832fc44d1ab12720, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732536370942 2024-11-25T12:06:13,514 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5393cf63d8c48da9b8a3e3e6e45d4d6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732536370942 2024-11-25T12:06:13,520 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting bb465d04a92d44e58d653388f78c7c3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732536372170 2024-11-25T12:06:13,520 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4971a8a46a9143b9a7c9ca8f83b7c92f, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732536372170 2024-11-25T12:06:13,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112584c8244009a340ab890034a8943baeaa_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536373508/Put/seqid=0 2024-11-25T12:06:13,562 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#C#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:06:13,563 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/02c6e32794c64200b1ab4d323df8849a is 50, key is test_row_0/C:col10/1732536372829/Put/seqid=0 2024-11-25T12:06:13,565 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T12:06:13,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-25T12:06:13,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:13,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:13,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:13,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:13,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:13,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:13,599 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241125ee6daa8e54c74f758e971f4f8a6327c2_23c89a604d5eddf398c6d709d0d8bf0d store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:13,601 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241125ee6daa8e54c74f758e971f4f8a6327c2_23c89a604d5eddf398c6d709d0d8bf0d, store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:13,601 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ee6daa8e54c74f758e971f4f8a6327c2_23c89a604d5eddf398c6d709d0d8bf0d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:13,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742436_1612 (size=14994) 2024-11-25T12:06:13,613 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:13,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536433607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:13,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536433609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:13,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:13,625 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112584c8244009a340ab890034a8943baeaa_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112584c8244009a340ab890034a8943baeaa_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:13,628 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/1596b69125d14dfe9f95250b66af3dfe, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:13,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/1596b69125d14dfe9f95250b66af3dfe is 175, key is test_row_0/A:col10/1732536373508/Put/seqid=0 2024-11-25T12:06:13,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742437_1613 (size=13017) 2024-11-25T12:06:13,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742438_1614 (size=4469) 2024-11-25T12:06:13,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742439_1615 (size=39949) 2024-11-25T12:06:13,667 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/1596b69125d14dfe9f95250b66af3dfe 2024-11-25T12:06:13,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/f398fcade18d419abfd731d391300de9 is 50, key is test_row_0/B:col10/1732536373508/Put/seqid=0 2024-11-25T12:06:13,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742440_1616 (size=12301) 2024-11-25T12:06:13,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536433726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:13,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:13,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536433727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:13,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-25T12:06:13,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:13,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:13,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:13,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:13,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:13,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:13,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/f398fcade18d419abfd731d391300de9 2024-11-25T12:06:13,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:13,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:13,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/3e2f6f3028de4f089eba67b28a1a11eb is 50, key is test_row_0/C:col10/1732536373508/Put/seqid=0 2024-11-25T12:06:13,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742441_1617 (size=12301) 2024-11-25T12:06:13,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/3e2f6f3028de4f089eba67b28a1a11eb 2024-11-25T12:06:13,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-25T12:06:13,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:13,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:13,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:13,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:13,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:13,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:13,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/1596b69125d14dfe9f95250b66af3dfe as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/1596b69125d14dfe9f95250b66af3dfe 2024-11-25T12:06:13,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/1596b69125d14dfe9f95250b66af3dfe, entries=200, sequenceid=328, filesize=39.0 K 2024-11-25T12:06:13,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:13,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536433936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:13,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:13,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:13,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:13,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/f398fcade18d419abfd731d391300de9 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/f398fcade18d419abfd731d391300de9 2024-11-25T12:06:13,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:13,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536433949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:13,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:13,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:13,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:13,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/f398fcade18d419abfd731d391300de9, entries=150, sequenceid=328, filesize=12.0 K 2024-11-25T12:06:13,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/3e2f6f3028de4f089eba67b28a1a11eb as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/3e2f6f3028de4f089eba67b28a1a11eb 2024-11-25T12:06:13,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/3e2f6f3028de4f089eba67b28a1a11eb, entries=150, sequenceid=328, filesize=12.0 K 2024-11-25T12:06:13,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 23c89a604d5eddf398c6d709d0d8bf0d in 468ms, sequenceid=328, compaction requested=true 2024-11-25T12:06:13,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:13,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:A, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:06:13,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:06:13,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:B, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:06:13,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:06:13,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:06:13,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-25T12:06:14,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-25T12:06:14,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:14,053 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-25T12:06:14,066 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#A#compaction#511 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:06:14,066 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/4a3b570759b34ce8ac67ac7b49339c2c is 175, key is test_row_0/A:col10/1732536372829/Put/seqid=0 2024-11-25T12:06:14,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T12:06:14,077 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/02c6e32794c64200b1ab4d323df8849a as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/02c6e32794c64200b1ab4d323df8849a 2024-11-25T12:06:14,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125065eda11d3754862b50849b81cdcb5ee_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536373593/Put/seqid=0 2024-11-25T12:06:14,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742442_1618 (size=31971) 2024-11-25T12:06:14,125 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/C of 23c89a604d5eddf398c6d709d0d8bf0d into 02c6e32794c64200b1ab4d323df8849a(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:14,125 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:14,125 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/C, priority=13, startTime=1732536373495; duration=0sec 2024-11-25T12:06:14,125 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-25T12:06:14,125 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:C 2024-11-25T12:06:14,125 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-25T12:06:14,132 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T12:06:14,132 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T12:06:14,132 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. because compaction request was cancelled 2024-11-25T12:06:14,132 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:A 2024-11-25T12:06:14,132 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:C 2024-11-25T12:06:14,132 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:06:14,137 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/4a3b570759b34ce8ac67ac7b49339c2c as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4a3b570759b34ce8ac67ac7b49339c2c 2024-11-25T12:06:14,141 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:06:14,141 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/B is initiating minor compaction (all files) 2024-11-25T12:06:14,141 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/B in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:14,141 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/03bff1b1f4364765a81989b38609e6e3, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/2b9725b34b954355aed6c4cdc559dd91, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/3db00a56dd32413d92d6d63e4a735377, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/f398fcade18d419abfd731d391300de9] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=48.7 K 2024-11-25T12:06:14,143 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 03bff1b1f4364765a81989b38609e6e3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732536370191 2024-11-25T12:06:14,143 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b9725b34b954355aed6c4cdc559dd91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732536370942 2024-11-25T12:06:14,146 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 3db00a56dd32413d92d6d63e4a735377, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732536372170 2024-11-25T12:06:14,146 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting f398fcade18d419abfd731d391300de9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732536372856 2024-11-25T12:06:14,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742443_1619 (size=12454) 2024-11-25T12:06:14,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:14,156 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125065eda11d3754862b50849b81cdcb5ee_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125065eda11d3754862b50849b81cdcb5ee_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:14,157 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/A of 23c89a604d5eddf398c6d709d0d8bf0d into 4a3b570759b34ce8ac67ac7b49339c2c(size=31.2 K), total size for store is 70.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:14,157 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:14,157 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/A, priority=13, startTime=1732536373495; duration=0sec 2024-11-25T12:06:14,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/2325c87e381a4cd787c607fb9244bef9, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:14,157 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:06:14,157 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:A 2024-11-25T12:06:14,157 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-25T12:06:14,157 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T12:06:14,157 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T12:06:14,157 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. because compaction request was cancelled 2024-11-25T12:06:14,157 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:B 2024-11-25T12:06:14,157 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-25T12:06:14,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/2325c87e381a4cd787c607fb9244bef9 is 175, key is test_row_0/A:col10/1732536373593/Put/seqid=0 2024-11-25T12:06:14,158 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-25T12:06:14,158 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-25T12:06:14,158 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. because compaction request was cancelled 2024-11-25T12:06:14,158 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:C 2024-11-25T12:06:14,161 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#B#compaction#515 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:06:14,162 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/51586b96372845caa07757c03b14ec53 is 50, key is test_row_0/B:col10/1732536373508/Put/seqid=0 2024-11-25T12:06:14,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742445_1621 (size=13051) 2024-11-25T12:06:14,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742444_1620 (size=31255) 2024-11-25T12:06:14,233 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/51586b96372845caa07757c03b14ec53 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/51586b96372845caa07757c03b14ec53 2024-11-25T12:06:14,236 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/2325c87e381a4cd787c607fb9244bef9 2024-11-25T12:06:14,241 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/B of 23c89a604d5eddf398c6d709d0d8bf0d into 51586b96372845caa07757c03b14ec53(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:14,241 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:14,241 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/B, priority=12, startTime=1732536373981; duration=0sec 2024-11-25T12:06:14,241 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:06:14,241 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:B 2024-11-25T12:06:14,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/bec37c63837c470d9debef51966935be is 50, key is test_row_0/B:col10/1732536373593/Put/seqid=0 2024-11-25T12:06:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:14,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:14,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742446_1622 (size=12301) 2024-11-25T12:06:14,300 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/bec37c63837c470d9debef51966935be 2024-11-25T12:06:14,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536434347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:14,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536434349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:14,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:14,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:14,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:14,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:14,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:14,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:14,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/3dbf24b6fc6e4692bcb6e03f2ff603eb is 50, key is test_row_0/C:col10/1732536373593/Put/seqid=0 2024-11-25T12:06:14,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742447_1623 (size=12301) 2024-11-25T12:06:14,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536434469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:14,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:14,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:14,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:14,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:14,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536434471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:14,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:14,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:14,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:14,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536434689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:14,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:14,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:14,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536434689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:14,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:14,849 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/3dbf24b6fc6e4692bcb6e03f2ff603eb 2024-11-25T12:06:14,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/2325c87e381a4cd787c607fb9244bef9 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/2325c87e381a4cd787c607fb9244bef9 2024-11-25T12:06:14,917 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/2325c87e381a4cd787c607fb9244bef9, entries=150, sequenceid=344, filesize=30.5 K 2024-11-25T12:06:14,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/bec37c63837c470d9debef51966935be as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/bec37c63837c470d9debef51966935be 2024-11-25T12:06:14,958 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/bec37c63837c470d9debef51966935be, entries=150, sequenceid=344, filesize=12.0 K 2024-11-25T12:06:14,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/3dbf24b6fc6e4692bcb6e03f2ff603eb as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/3dbf24b6fc6e4692bcb6e03f2ff603eb 2024-11-25T12:06:14,963 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/3dbf24b6fc6e4692bcb6e03f2ff603eb, entries=150, sequenceid=344, filesize=12.0 K 2024-11-25T12:06:14,966 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 23c89a604d5eddf398c6d709d0d8bf0d in 912ms, sequenceid=344, compaction requested=true 2024-11-25T12:06:14,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:14,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:14,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-25T12:06:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-25T12:06:14,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-25T12:06:14,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0300 sec 2024-11-25T12:06:14,974 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.0360 sec 2024-11-25T12:06:15,002 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:06:15,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-25T12:06:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:15,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ae9ad265f39945c88faf4311584c77b7_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536374999/Put/seqid=0 2024-11-25T12:06:15,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742448_1624 (size=14994) 2024-11-25T12:06:15,061 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:15,065 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125ae9ad265f39945c88faf4311584c77b7_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ae9ad265f39945c88faf4311584c77b7_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:15,067 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/0dd7cba111e74bcda12b469d9b3317b9, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:15,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/0dd7cba111e74bcda12b469d9b3317b9 is 175, key is test_row_0/A:col10/1732536374999/Put/seqid=0 2024-11-25T12:06:15,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536435063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:15,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:15,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536435067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:15,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:15,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:15,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-25T12:06:15,092 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:06:15,093 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-25T12:06:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-25T12:06:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T12:06:15,097 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-25T12:06:15,097 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-25T12:06:15,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-25T12:06:15,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742449_1625 (size=39949) 2024-11-25T12:06:15,113 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/0dd7cba111e74bcda12b469d9b3317b9 2024-11-25T12:06:15,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/65e95a59a24c466db84410b21a98c3e0 is 50, key is test_row_0/B:col10/1732536374999/Put/seqid=0 2024-11-25T12:06:15,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742450_1626 (size=12301) 2024-11-25T12:06:15,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/65e95a59a24c466db84410b21a98c3e0 2024-11-25T12:06:15,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536435181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:15,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:15,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:15,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:15,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536435181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:15,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/95b906fffbac4349a0e8c116f13b01ac is 50, key is test_row_0/C:col10/1732536374999/Put/seqid=0 2024-11-25T12:06:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T12:06:15,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742451_1627 (size=12301) 2024-11-25T12:06:15,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/95b906fffbac4349a0e8c116f13b01ac 2024-11-25T12:06:15,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/0dd7cba111e74bcda12b469d9b3317b9 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/0dd7cba111e74bcda12b469d9b3317b9 2024-11-25T12:06:15,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T12:06:15,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/0dd7cba111e74bcda12b469d9b3317b9, entries=200, sequenceid=368, filesize=39.0 K 2024-11-25T12:06:15,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/65e95a59a24c466db84410b21a98c3e0 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/65e95a59a24c466db84410b21a98c3e0 2024-11-25T12:06:15,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:15,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:15,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:15,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:15,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:15,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/65e95a59a24c466db84410b21a98c3e0, entries=150, sequenceid=368, filesize=12.0 K 2024-11-25T12:06:15,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/95b906fffbac4349a0e8c116f13b01ac as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/95b906fffbac4349a0e8c116f13b01ac 2024-11-25T12:06:15,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/95b906fffbac4349a0e8c116f13b01ac, entries=150, sequenceid=368, filesize=12.0 K 2024-11-25T12:06:15,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 23c89a604d5eddf398c6d709d0d8bf0d in 279ms, sequenceid=368, compaction requested=true 2024-11-25T12:06:15,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:15,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:A, priority=-2147483648, current under compaction store size is 1 2024-11-25T12:06:15,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:06:15,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:B, priority=-2147483648, current under compaction store size is 2 2024-11-25T12:06:15,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-25T12:06:15,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 23c89a604d5eddf398c6d709d0d8bf0d:C, priority=-2147483648, current under compaction store size is 3 2024-11-25T12:06:15,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-25T12:06:15,283 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:06:15,284 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:06:15,284 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/A is initiating minor compaction (all files) 2024-11-25T12:06:15,284 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/A in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:15,284 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T12:06:15,284 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4a3b570759b34ce8ac67ac7b49339c2c, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/1596b69125d14dfe9f95250b66af3dfe, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/2325c87e381a4cd787c607fb9244bef9, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/0dd7cba111e74bcda12b469d9b3317b9] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=139.8 K 2024-11-25T12:06:15,284 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:15,285 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. files: [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4a3b570759b34ce8ac67ac7b49339c2c, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/1596b69125d14dfe9f95250b66af3dfe, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/2325c87e381a4cd787c607fb9244bef9, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/0dd7cba111e74bcda12b469d9b3317b9] 2024-11-25T12:06:15,285 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a3b570759b34ce8ac67ac7b49339c2c, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732536372170 2024-11-25T12:06:15,285 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 1596b69125d14dfe9f95250b66af3dfe, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732536372856 2024-11-25T12:06:15,286 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 2325c87e381a4cd787c607fb9244bef9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732536373593 2024-11-25T12:06:15,286 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dd7cba111e74bcda12b469d9b3317b9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732536374334 2024-11-25T12:06:15,287 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-25T12:06:15,287 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/C is initiating minor compaction (all files) 2024-11-25T12:06:15,287 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/C in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:15,287 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/02c6e32794c64200b1ab4d323df8849a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/3e2f6f3028de4f089eba67b28a1a11eb, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/3dbf24b6fc6e4692bcb6e03f2ff603eb, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/95b906fffbac4349a0e8c116f13b01ac] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=48.8 K 2024-11-25T12:06:15,288 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02c6e32794c64200b1ab4d323df8849a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732536372170 2024-11-25T12:06:15,288 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e2f6f3028de4f089eba67b28a1a11eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732536372856 2024-11-25T12:06:15,289 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dbf24b6fc6e4692bcb6e03f2ff603eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732536373593 2024-11-25T12:06:15,289 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95b906fffbac4349a0e8c116f13b01ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732536374337 2024-11-25T12:06:15,297 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:15,307 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#C#compaction#522 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:06:15,307 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/fcc51ac59a02461e96497351f9d824b1 is 50, key is test_row_0/C:col10/1732536374999/Put/seqid=0 2024-11-25T12:06:15,322 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112525fb6e28af06408bad90209e999dbb2e_23c89a604d5eddf398c6d709d0d8bf0d store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:15,325 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112525fb6e28af06408bad90209e999dbb2e_23c89a604d5eddf398c6d709d0d8bf0d, store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:15,325 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112525fb6e28af06408bad90209e999dbb2e_23c89a604d5eddf398c6d709d0d8bf0d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:15,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742452_1628 (size=13153) 2024-11-25T12:06:15,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742453_1629 (size=4469) 2024-11-25T12:06:15,371 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/fcc51ac59a02461e96497351f9d824b1 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/fcc51ac59a02461e96497351f9d824b1 2024-11-25T12:06:15,378 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/C of 23c89a604d5eddf398c6d709d0d8bf0d into fcc51ac59a02461e96497351f9d824b1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:15,378 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:15,378 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/C, priority=12, startTime=1732536375281; duration=0sec 2024-11-25T12:06:15,378 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-25T12:06:15,379 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:C 2024-11-25T12:06:15,380 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-25T12:06:15,385 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-25T12:06:15,385 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1541): 23c89a604d5eddf398c6d709d0d8bf0d/B is initiating minor compaction (all files) 2024-11-25T12:06:15,385 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 23c89a604d5eddf398c6d709d0d8bf0d/B in TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:15,385 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/51586b96372845caa07757c03b14ec53, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/bec37c63837c470d9debef51966935be, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/65e95a59a24c466db84410b21a98c3e0] into tmpdir=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp, totalSize=36.8 K 2024-11-25T12:06:15,387 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51586b96372845caa07757c03b14ec53, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732536372856 2024-11-25T12:06:15,387 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting bec37c63837c470d9debef51966935be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732536373593 2024-11-25T12:06:15,388 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65e95a59a24c466db84410b21a98c3e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732536374337 2024-11-25T12:06:15,400 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-25T12:06:15,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-25T12:06:15,401 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#B#compaction#523 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:06:15,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:15,401 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/e15427103cd24a9e8b8d02828ad07a89 is 50, key is test_row_0/B:col10/1732536374999/Put/seqid=0 2024-11-25T12:06:15,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T12:06:15,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:15,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:15,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:15,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:15,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T12:06:15,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d3442a81b134465c98fb9d5b07967e18_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536375046/Put/seqid=0 2024-11-25T12:06:15,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742454_1630 (size=13153) 2024-11-25T12:06:15,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742455_1631 (size=14994) 2024-11-25T12:06:15,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:15,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536435457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:15,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:15,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536435460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:15,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:15,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:15,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:15,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T12:06:15,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:15,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:15,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:15,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:15,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:15,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536435566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:15,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536435576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:15,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:15,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T12:06:15,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:15,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:15,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:15,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:15,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:15,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T12:06:15,766 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 23c89a604d5eddf398c6d709d0d8bf0d#A#compaction#521 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T12:06:15,767 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/b905d1b6476e4c1fa81a805f5ea036b6 is 175, key is test_row_0/A:col10/1732536374999/Put/seqid=0 2024-11-25T12:06:15,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:15,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536435776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:15,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:15,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:15,783 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:15,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536435786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:15,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742456_1632 (size=32107) 2024-11-25T12:06:15,828 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/b905d1b6476e4c1fa81a805f5ea036b6 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/b905d1b6476e4c1fa81a805f5ea036b6 2024-11-25T12:06:15,864 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:15,865 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/A of 23c89a604d5eddf398c6d709d0d8bf0d into b905d1b6476e4c1fa81a805f5ea036b6(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:15,866 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:15,866 INFO [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/A, priority=12, startTime=1732536375281; duration=0sec 2024-11-25T12:06:15,866 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:06:15,866 DEBUG [RS:0;2bf237fd86ca:37643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:A 2024-11-25T12:06:15,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T12:06:15,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:15,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:15,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:15,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:15,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:15,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:15,909 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125d3442a81b134465c98fb9d5b07967e18_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d3442a81b134465c98fb9d5b07967e18_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:15,913 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/e15427103cd24a9e8b8d02828ad07a89 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/e15427103cd24a9e8b8d02828ad07a89 2024-11-25T12:06:15,917 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/f211da3a7240418f9255d4f1b71dcbd5, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:15,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/f211da3a7240418f9255d4f1b71dcbd5 is 175, key is test_row_0/A:col10/1732536375046/Put/seqid=0 2024-11-25T12:06:15,934 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 23c89a604d5eddf398c6d709d0d8bf0d/B of 23c89a604d5eddf398c6d709d0d8bf0d into e15427103cd24a9e8b8d02828ad07a89(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T12:06:15,934 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:15,934 INFO [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., storeName=23c89a604d5eddf398c6d709d0d8bf0d/B, priority=13, startTime=1732536375281; duration=0sec 2024-11-25T12:06:15,934 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-25T12:06:15,934 DEBUG [RS:0;2bf237fd86ca:37643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 23c89a604d5eddf398c6d709d0d8bf0d:B 2024-11-25T12:06:15,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742457_1633 (size=39949) 2024-11-25T12:06:16,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T12:06:16,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:16,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:16,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:16,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:16,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:16,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:16,061 INFO [Thread-2364 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2364. 2024-11-25T12:06:16,062 DEBUG [Thread-2364 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:06:16,062 DEBUG [Thread-2364 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:16,062 INFO [Thread-2366 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2366. 2024-11-25T12:06:16,062 DEBUG [Thread-2366 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:06:16,062 DEBUG [Thread-2366 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:16,062 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:06:16,062 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:06:16,062 DEBUG [Thread-2366 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:16,062 DEBUG [Thread-2364 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:16,066 INFO [Thread-2362 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2362. 2024-11-25T12:06:16,066 DEBUG [Thread-2362 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:06:16,066 DEBUG [Thread-2362 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:16,066 DEBUG [Thread-2362 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:16,066 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:06:16,069 INFO [Thread-2370 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2370. 2024-11-25T12:06:16,069 DEBUG [Thread-2370 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:06:16,069 DEBUG [Thread-2370 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:16,069 DEBUG [Thread-2370 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:16,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:06:16,070 INFO [Thread-2368 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2368. 2024-11-25T12:06:16,070 DEBUG [Thread-2368 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:06:16,070 DEBUG [Thread-2368 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:16,070 DEBUG [Thread-2368 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:16,070 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:06:16,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:16,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536436097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536436097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:16,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T12:06:16,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:16,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:16,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:16,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:16,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T12:06:16,340 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=386, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/f211da3a7240418f9255d4f1b71dcbd5 2024-11-25T12:06:16,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/c6d060684bf94d899b300bb3de0227c3 is 50, key is test_row_0/B:col10/1732536375046/Put/seqid=0 2024-11-25T12:06:16,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742458_1634 (size=12301) 2024-11-25T12:06:16,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T12:06:16,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:16,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:16,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:16,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:16,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:16,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:16,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T12:06:16,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:16,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:16,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:16,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:16,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:16,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:16,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:16,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-25T12:06:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732536436607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37643 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60028 deadline: 1732536436607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:16,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:16,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 , the old value is region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:16,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:16,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T12:06:16,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:16,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., hostname=2bf237fd86ca,37643,1732536214674, seqNum=5 because the exception is null or not the one we care about 2024-11-25T12:06:16,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T12:06:16,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:16,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:16,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:16,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:16,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:16,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:16,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/c6d060684bf94d899b300bb3de0227c3 2024-11-25T12:06:16,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/8f5b87a1aaab4b34af906c8bbc5a9507 is 50, key is test_row_0/C:col10/1732536375046/Put/seqid=0 2024-11-25T12:06:16,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742459_1635 (size=12301) 2024-11-25T12:06:16,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T12:06:16,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:16,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:16,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:16,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:16,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:16,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:16,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T12:06:16,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:16,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:16,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:16,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:16,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:16,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:17,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T12:06:17,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:17,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:17,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:17,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:17,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T12:06:17,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/8f5b87a1aaab4b34af906c8bbc5a9507 2024-11-25T12:06:17,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/f211da3a7240418f9255d4f1b71dcbd5 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/f211da3a7240418f9255d4f1b71dcbd5 2024-11-25T12:06:17,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/f211da3a7240418f9255d4f1b71dcbd5, entries=200, sequenceid=386, filesize=39.0 K 2024-11-25T12:06:17,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/c6d060684bf94d899b300bb3de0227c3 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/c6d060684bf94d899b300bb3de0227c3 2024-11-25T12:06:17,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/c6d060684bf94d899b300bb3de0227c3, entries=150, sequenceid=386, filesize=12.0 K 2024-11-25T12:06:17,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/8f5b87a1aaab4b34af906c8bbc5a9507 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/8f5b87a1aaab4b34af906c8bbc5a9507 2024-11-25T12:06:17,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/8f5b87a1aaab4b34af906c8bbc5a9507, entries=150, sequenceid=386, filesize=12.0 K 2024-11-25T12:06:17,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 23c89a604d5eddf398c6d709d0d8bf0d in 1786ms, sequenceid=386, compaction requested=false 2024-11-25T12:06:17,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:17,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T12:06:17,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37643 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-25T12:06:17,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:17,282 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-25T12:06:17,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f4fe1229ba044b1abd1da40f5b154815_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536375455/Put/seqid=0 2024-11-25T12:06:17,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742460_1636 (size=12454) 2024-11-25T12:06:17,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37643 {}] regionserver.HRegion(8855): Flush requested on 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:17,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. as already flushing 2024-11-25T12:06:17,619 INFO [Thread-2359 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2359. 2024-11-25T12:06:17,619 INFO [Thread-2355 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2355. 2024-11-25T12:06:17,619 DEBUG [Thread-2359 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:06:17,619 DEBUG [Thread-2355 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:06:17,619 DEBUG [Thread-2359 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:17,619 DEBUG [Thread-2355 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:17,619 DEBUG [Thread-2355 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:17,619 DEBUG [Thread-2359 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:17,619 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:06:17,619 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:06:17,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:17,694 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241125f4fe1229ba044b1abd1da40f5b154815_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f4fe1229ba044b1abd1da40f5b154815_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:17,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/64565f18aaef4b42a405301fefe37141, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:17,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/64565f18aaef4b42a405301fefe37141 is 175, key is test_row_0/A:col10/1732536375455/Put/seqid=0 2024-11-25T12:06:17,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742461_1637 (size=31255) 2024-11-25T12:06:18,100 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=407, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/64565f18aaef4b42a405301fefe37141 2024-11-25T12:06:18,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/05195521d6ba490fb6b9c864da3dd278 is 50, key is test_row_0/B:col10/1732536375455/Put/seqid=0 2024-11-25T12:06:18,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742462_1638 (size=12301) 2024-11-25T12:06:18,522 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/05195521d6ba490fb6b9c864da3dd278 2024-11-25T12:06:18,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/343f6ce4f7c440019cf818243ea17656 is 50, key is test_row_0/C:col10/1732536375455/Put/seqid=0 2024-11-25T12:06:18,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742463_1639 (size=12301) 2024-11-25T12:06:18,534 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/343f6ce4f7c440019cf818243ea17656 2024-11-25T12:06:18,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/64565f18aaef4b42a405301fefe37141 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/64565f18aaef4b42a405301fefe37141 2024-11-25T12:06:18,541 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/64565f18aaef4b42a405301fefe37141, entries=150, sequenceid=407, filesize=30.5 K 2024-11-25T12:06:18,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/05195521d6ba490fb6b9c864da3dd278 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/05195521d6ba490fb6b9c864da3dd278 2024-11-25T12:06:18,546 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/05195521d6ba490fb6b9c864da3dd278, entries=150, sequenceid=407, filesize=12.0 K 2024-11-25T12:06:18,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/343f6ce4f7c440019cf818243ea17656 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/343f6ce4f7c440019cf818243ea17656 2024-11-25T12:06:18,549 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/343f6ce4f7c440019cf818243ea17656, entries=150, sequenceid=407, filesize=12.0 K 2024-11-25T12:06:18,550 INFO [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=13.42 KB/13740 for 23c89a604d5eddf398c6d709d0d8bf0d in 1268ms, sequenceid=407, compaction requested=true 2024-11-25T12:06:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 23c89a604d5eddf398c6d709d0d8bf0d: 2024-11-25T12:06:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2bf237fd86ca:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-25T12:06:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-25T12:06:18,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-25T12:06:18,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4540 sec 2024-11-25T12:06:18,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 3.4610 sec 2024-11-25T12:06:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-25T12:06:19,256 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:06:21,858 INFO [Thread-2357 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2357. 2024-11-25T12:06:21,858 DEBUG [Thread-2357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:06:21,858 DEBUG [Thread-2357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:21,858 DEBUG [Thread-2357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:21,858 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:06:21,868 INFO [Thread-2353 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2353. 2024-11-25T12:06:21,868 DEBUG [Thread-2353 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:06:21,868 DEBUG [Thread-2353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:21,868 DEBUG [Thread-2353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:21,869 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:06:21,898 INFO [Thread-2351 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2351. 2024-11-25T12:06:21,898 DEBUG [Thread-2351 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-25T12:06:21,898 DEBUG [Thread-2351 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:21,898 DEBUG [Thread-2351 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:21,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:06:21,898 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-25T12:06:21,898 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-25T12:06:21,898 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-25T12:06:21,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-11-25T12:06:21,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-25T12:06:21,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-11-25T12:06:21,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-25T12:06:21,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2121 2024-11-25T12:06:21,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2021 2024-11-25T12:06:21,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1968 2024-11-25T12:06:21,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2087 2024-11-25T12:06:21,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2033 2024-11-25T12:06:21,899 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-25T12:06:21,899 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T12:06:21,899 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:06:21,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:21,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:21,899 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:06:21,900 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-25T12:06:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-25T12:06:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T12:06:21,904 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536381903"}]},"ts":"1732536381903"} 2024-11-25T12:06:21,905 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-25T12:06:21,905 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-25T12:06:21,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-25T12:06:21,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23c89a604d5eddf398c6d709d0d8bf0d, UNASSIGN}] 2024-11-25T12:06:21,910 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23c89a604d5eddf398c6d709d0d8bf0d, UNASSIGN 2024-11-25T12:06:21,911 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=158 updating hbase:meta row=23c89a604d5eddf398c6d709d0d8bf0d, regionState=CLOSING, regionLocation=2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:21,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=23c89a604d5eddf398c6d709d0d8bf0d, UNASSIGN because future has completed 2024-11-25T12:06:21,916 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T12:06:21,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; CloseRegionProcedure 23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674}] 2024-11-25T12:06:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T12:06:22,068 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(122): Close 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:22,068 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T12:06:22,069 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1722): Closing 23c89a604d5eddf398c6d709d0d8bf0d, disabling compactions & flushes 2024-11-25T12:06:22,069 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:22,069 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:22,069 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. after waiting 0 ms 2024-11-25T12:06:22,069 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:22,069 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(2902): Flushing 23c89a604d5eddf398c6d709d0d8bf0d 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-25T12:06:22,081 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411258699174038d14ee7bd09801d002653ad_23c89a604d5eddf398c6d709d0d8bf0d is 50, key is test_row_0/A:col10/1732536381867/Put/seqid=0 2024-11-25T12:06:22,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742464_1640 (size=12454) 2024-11-25T12:06:22,090 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T12:06:22,101 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411258699174038d14ee7bd09801d002653ad_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258699174038d14ee7bd09801d002653ad_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:22,102 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/de7d5fdcb67241a7b22b8632fce48241, store: [table=TestAcidGuarantees family=A region=23c89a604d5eddf398c6d709d0d8bf0d] 2024-11-25T12:06:22,103 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/de7d5fdcb67241a7b22b8632fce48241 is 175, key is test_row_0/A:col10/1732536381867/Put/seqid=0 2024-11-25T12:06:22,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742465_1641 (size=31255) 2024-11-25T12:06:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T12:06:22,511 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=415, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/de7d5fdcb67241a7b22b8632fce48241 2024-11-25T12:06:22,517 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/d02295e299124b5aad496461e703fcb6 is 50, key is test_row_0/B:col10/1732536381867/Put/seqid=0 2024-11-25T12:06:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T12:06:22,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742466_1642 (size=12301) 2024-11-25T12:06:22,933 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/d02295e299124b5aad496461e703fcb6 2024-11-25T12:06:22,940 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/0573f5ba2feb49f389b0e1fc7555921e is 50, key is test_row_0/C:col10/1732536381867/Put/seqid=0 2024-11-25T12:06:22,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742467_1643 (size=12301) 2024-11-25T12:06:22,945 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/0573f5ba2feb49f389b0e1fc7555921e 2024-11-25T12:06:22,960 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/A/de7d5fdcb67241a7b22b8632fce48241 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/de7d5fdcb67241a7b22b8632fce48241 2024-11-25T12:06:22,964 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/de7d5fdcb67241a7b22b8632fce48241, entries=150, sequenceid=415, filesize=30.5 K 2024-11-25T12:06:22,965 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/B/d02295e299124b5aad496461e703fcb6 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/d02295e299124b5aad496461e703fcb6 2024-11-25T12:06:22,975 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/d02295e299124b5aad496461e703fcb6, entries=150, sequenceid=415, filesize=12.0 K 2024-11-25T12:06:22,976 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/.tmp/C/0573f5ba2feb49f389b0e1fc7555921e as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/0573f5ba2feb49f389b0e1fc7555921e 2024-11-25T12:06:22,979 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/0573f5ba2feb49f389b0e1fc7555921e, entries=150, sequenceid=415, filesize=12.0 K 2024-11-25T12:06:22,980 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 23c89a604d5eddf398c6d709d0d8bf0d in 911ms, sequenceid=415, compaction requested=true 2024-11-25T12:06:22,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4aeab5bec96b4b16b951291861b885cc, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/aa6d139ab7434d4083cc6befdd0554d5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/a714950c338243c0b3987a0b19d296df, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/01f872f389344c3f82744dd4bde9863f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/9d41cba9f1e84291829e89102581a3c0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/ccb5a73801674d298e618b5e1fee0912, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4acccadd6dfd4a06b60a1dacba77267f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/c18be4f3610948088a10e58583b68e0d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7b023d652d9f4560b3a680afde3d8d52, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7e8d60930cff43dc82ce428c4946ee4e, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7f88517d91d34e46a48fc903e1221f44, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/58be2d86357d449ea0f7aa67e026d521, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/63d5b1f3887e4c7ca46d1c84b6b586ec, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/fe621cc5c0684866836c539f9c74cbf5, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/02ccf7e606434494b1fe29a73752c4f7, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/74adf05786a84d03ab402b4be57d34ba, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/18007c66f7514043b1219c607ce651e1, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/921e5f2401034ac98733c743a1485f92, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/45d995ff357644ce87ac2fecdbb13f73, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/d5393cf63d8c48da9b8a3e3e6e45d4d6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4971a8a46a9143b9a7c9ca8f83b7c92f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4a3b570759b34ce8ac67ac7b49339c2c, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/1596b69125d14dfe9f95250b66af3dfe, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/2325c87e381a4cd787c607fb9244bef9, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/0dd7cba111e74bcda12b469d9b3317b9] to archive 2024-11-25T12:06:22,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:06:22,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4aeab5bec96b4b16b951291861b885cc to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4aeab5bec96b4b16b951291861b885cc 2024-11-25T12:06:22,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/aa6d139ab7434d4083cc6befdd0554d5 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/aa6d139ab7434d4083cc6befdd0554d5 2024-11-25T12:06:22,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/a714950c338243c0b3987a0b19d296df to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/a714950c338243c0b3987a0b19d296df 2024-11-25T12:06:22,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/01f872f389344c3f82744dd4bde9863f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/01f872f389344c3f82744dd4bde9863f 2024-11-25T12:06:22,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/9d41cba9f1e84291829e89102581a3c0 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/9d41cba9f1e84291829e89102581a3c0 2024-11-25T12:06:22,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/ccb5a73801674d298e618b5e1fee0912 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/ccb5a73801674d298e618b5e1fee0912 2024-11-25T12:06:22,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4acccadd6dfd4a06b60a1dacba77267f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4acccadd6dfd4a06b60a1dacba77267f 2024-11-25T12:06:22,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/c18be4f3610948088a10e58583b68e0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/c18be4f3610948088a10e58583b68e0d 2024-11-25T12:06:22,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7b023d652d9f4560b3a680afde3d8d52 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7b023d652d9f4560b3a680afde3d8d52 2024-11-25T12:06:23,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7e8d60930cff43dc82ce428c4946ee4e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7e8d60930cff43dc82ce428c4946ee4e 2024-11-25T12:06:23,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7f88517d91d34e46a48fc903e1221f44 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/7f88517d91d34e46a48fc903e1221f44 2024-11-25T12:06:23,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/58be2d86357d449ea0f7aa67e026d521 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/58be2d86357d449ea0f7aa67e026d521 2024-11-25T12:06:23,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/63d5b1f3887e4c7ca46d1c84b6b586ec to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/63d5b1f3887e4c7ca46d1c84b6b586ec 2024-11-25T12:06:23,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/fe621cc5c0684866836c539f9c74cbf5 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/fe621cc5c0684866836c539f9c74cbf5 2024-11-25T12:06:23,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/02ccf7e606434494b1fe29a73752c4f7 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/02ccf7e606434494b1fe29a73752c4f7 2024-11-25T12:06:23,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/74adf05786a84d03ab402b4be57d34ba to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/74adf05786a84d03ab402b4be57d34ba 2024-11-25T12:06:23,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/18007c66f7514043b1219c607ce651e1 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/18007c66f7514043b1219c607ce651e1 2024-11-25T12:06:23,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/921e5f2401034ac98733c743a1485f92 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/921e5f2401034ac98733c743a1485f92 2024-11-25T12:06:23,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/45d995ff357644ce87ac2fecdbb13f73 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/45d995ff357644ce87ac2fecdbb13f73 2024-11-25T12:06:23,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/d5393cf63d8c48da9b8a3e3e6e45d4d6 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/d5393cf63d8c48da9b8a3e3e6e45d4d6 2024-11-25T12:06:23,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4971a8a46a9143b9a7c9ca8f83b7c92f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4971a8a46a9143b9a7c9ca8f83b7c92f 2024-11-25T12:06:23,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4a3b570759b34ce8ac67ac7b49339c2c to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/4a3b570759b34ce8ac67ac7b49339c2c 2024-11-25T12:06:23,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/1596b69125d14dfe9f95250b66af3dfe to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/1596b69125d14dfe9f95250b66af3dfe 2024-11-25T12:06:23,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/2325c87e381a4cd787c607fb9244bef9 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/2325c87e381a4cd787c607fb9244bef9 2024-11-25T12:06:23,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/0dd7cba111e74bcda12b469d9b3317b9 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/0dd7cba111e74bcda12b469d9b3317b9 2024-11-25T12:06:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T12:06:23,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/ee5ca300044d485cbf1e08c01283fd80, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/985d5b75b8a541b6a50cff9a5c9389a6, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/610d9ccd39124204a531d75c687ac17d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/74dc19483ba041a99cfc1a483cbf1c69, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/a64a9c94df664c9a8abf25e5dbf20db2, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/0ab563e817644e3abaefe53e6bce4cec, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/ef9761a8369f4f8da9812426f38bfa49, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/6c2f2e5ed27e4189a7d543c5235a9783, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/e09e00d7a8bd4abba1717bc4d58f6fa0, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/d5d5dd0ee04a4e9e826268facf8e5727, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/39789e273399443fb45b0415a8389faa, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/a4a9981e6c3a43959d1336d67546064f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/39bfec65be5144c490458482cadeefcd, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/d1cbc11325074389a0b9fcb273e316ad, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/98d822cd6dea46d78b1803e3704aa925, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/59745b552eab44c7afc8228cb6e15e30, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/900d974f804f4a0791fd7eee1b704b54, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/03bff1b1f4364765a81989b38609e6e3, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/5ee64e981c2345f7ad7f7742141b67d7, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/2b9725b34b954355aed6c4cdc559dd91, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/3db00a56dd32413d92d6d63e4a735377, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/51586b96372845caa07757c03b14ec53, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/f398fcade18d419abfd731d391300de9, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/bec37c63837c470d9debef51966935be, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/65e95a59a24c466db84410b21a98c3e0] to archive 2024-11-25T12:06:23,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:06:23,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/ee5ca300044d485cbf1e08c01283fd80 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/ee5ca300044d485cbf1e08c01283fd80 2024-11-25T12:06:23,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/985d5b75b8a541b6a50cff9a5c9389a6 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/985d5b75b8a541b6a50cff9a5c9389a6 2024-11-25T12:06:23,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/610d9ccd39124204a531d75c687ac17d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/610d9ccd39124204a531d75c687ac17d 2024-11-25T12:06:23,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/74dc19483ba041a99cfc1a483cbf1c69 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/74dc19483ba041a99cfc1a483cbf1c69 2024-11-25T12:06:23,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/a64a9c94df664c9a8abf25e5dbf20db2 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/a64a9c94df664c9a8abf25e5dbf20db2 2024-11-25T12:06:23,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/0ab563e817644e3abaefe53e6bce4cec to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/0ab563e817644e3abaefe53e6bce4cec 2024-11-25T12:06:23,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/ef9761a8369f4f8da9812426f38bfa49 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/ef9761a8369f4f8da9812426f38bfa49 2024-11-25T12:06:23,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/6c2f2e5ed27e4189a7d543c5235a9783 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/6c2f2e5ed27e4189a7d543c5235a9783 2024-11-25T12:06:23,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/e09e00d7a8bd4abba1717bc4d58f6fa0 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/e09e00d7a8bd4abba1717bc4d58f6fa0 2024-11-25T12:06:23,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/d5d5dd0ee04a4e9e826268facf8e5727 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/d5d5dd0ee04a4e9e826268facf8e5727 2024-11-25T12:06:23,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/39789e273399443fb45b0415a8389faa to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/39789e273399443fb45b0415a8389faa 2024-11-25T12:06:23,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/a4a9981e6c3a43959d1336d67546064f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/a4a9981e6c3a43959d1336d67546064f 2024-11-25T12:06:23,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/39bfec65be5144c490458482cadeefcd to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/39bfec65be5144c490458482cadeefcd 2024-11-25T12:06:23,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/d1cbc11325074389a0b9fcb273e316ad to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/d1cbc11325074389a0b9fcb273e316ad 2024-11-25T12:06:23,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/98d822cd6dea46d78b1803e3704aa925 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/98d822cd6dea46d78b1803e3704aa925 2024-11-25T12:06:23,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/59745b552eab44c7afc8228cb6e15e30 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/59745b552eab44c7afc8228cb6e15e30 2024-11-25T12:06:23,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/900d974f804f4a0791fd7eee1b704b54 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/900d974f804f4a0791fd7eee1b704b54 2024-11-25T12:06:23,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/03bff1b1f4364765a81989b38609e6e3 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/03bff1b1f4364765a81989b38609e6e3 2024-11-25T12:06:23,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/5ee64e981c2345f7ad7f7742141b67d7 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/5ee64e981c2345f7ad7f7742141b67d7 2024-11-25T12:06:23,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/2b9725b34b954355aed6c4cdc559dd91 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/2b9725b34b954355aed6c4cdc559dd91 2024-11-25T12:06:23,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/3db00a56dd32413d92d6d63e4a735377 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/3db00a56dd32413d92d6d63e4a735377 2024-11-25T12:06:23,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/51586b96372845caa07757c03b14ec53 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/51586b96372845caa07757c03b14ec53 2024-11-25T12:06:23,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/f398fcade18d419abfd731d391300de9 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/f398fcade18d419abfd731d391300de9 2024-11-25T12:06:23,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/bec37c63837c470d9debef51966935be to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/bec37c63837c470d9debef51966935be 2024-11-25T12:06:23,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/65e95a59a24c466db84410b21a98c3e0 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/65e95a59a24c466db84410b21a98c3e0 2024-11-25T12:06:23,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/7fd22b535c984271a14ce875f6ba3459, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/d901b90d9e944c3597623c0e82eed247, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/81fb2d62d55a41c78bca4077ba8340fd, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/279d745fcaae4782b5a1538b3eefc0ba, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/2916c12220184ce6a55468bf38942a29, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/8548ff7a7d1f4c9c8b7eceae4ef12ebc, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/b30ee7424ea5476aa49ed132e821691f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/72e71991a2244c399faf00df71aeb52f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/ee4e015b9ca44b849e7f3791c9165203, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/85937814111e401eaa374bfbfacc0a84, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/09bb1f86d31b4773a080a66ce80a5c8f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/aa1168c4d10849659c16f59394d0314b, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/5caba28645454a24aadf3fb6d815103d, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/039cd58eb8c34eb092ff18fd5100ed5f, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/e3c269cd715e4f2c8ef137dc04f3c544, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/d326b49a5945424a978572c75765ab0c, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/4ea0cf819cc8450c8018212b6f73756a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/edf8185fb32e4caa9284ff8ebcdbbe82, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/ef2c25b4fbd34241b642bf871d050063, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/5eca799bdbdb49ea832fc44d1ab12720, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/02c6e32794c64200b1ab4d323df8849a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/bb465d04a92d44e58d653388f78c7c3a, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/3e2f6f3028de4f089eba67b28a1a11eb, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/3dbf24b6fc6e4692bcb6e03f2ff603eb, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/95b906fffbac4349a0e8c116f13b01ac] to archive 2024-11-25T12:06:23,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T12:06:23,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/7fd22b535c984271a14ce875f6ba3459 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/7fd22b535c984271a14ce875f6ba3459 2024-11-25T12:06:23,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/d901b90d9e944c3597623c0e82eed247 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/d901b90d9e944c3597623c0e82eed247 2024-11-25T12:06:23,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/81fb2d62d55a41c78bca4077ba8340fd to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/81fb2d62d55a41c78bca4077ba8340fd 2024-11-25T12:06:23,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/279d745fcaae4782b5a1538b3eefc0ba to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/279d745fcaae4782b5a1538b3eefc0ba 2024-11-25T12:06:23,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/2916c12220184ce6a55468bf38942a29 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/2916c12220184ce6a55468bf38942a29 2024-11-25T12:06:23,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/8548ff7a7d1f4c9c8b7eceae4ef12ebc to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/8548ff7a7d1f4c9c8b7eceae4ef12ebc 2024-11-25T12:06:23,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/b30ee7424ea5476aa49ed132e821691f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/b30ee7424ea5476aa49ed132e821691f 2024-11-25T12:06:23,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/72e71991a2244c399faf00df71aeb52f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/72e71991a2244c399faf00df71aeb52f 2024-11-25T12:06:23,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/ee4e015b9ca44b849e7f3791c9165203 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/ee4e015b9ca44b849e7f3791c9165203 2024-11-25T12:06:23,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/85937814111e401eaa374bfbfacc0a84 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/85937814111e401eaa374bfbfacc0a84 2024-11-25T12:06:23,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/09bb1f86d31b4773a080a66ce80a5c8f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/09bb1f86d31b4773a080a66ce80a5c8f 2024-11-25T12:06:23,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/aa1168c4d10849659c16f59394d0314b to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/aa1168c4d10849659c16f59394d0314b 2024-11-25T12:06:23,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/5caba28645454a24aadf3fb6d815103d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/5caba28645454a24aadf3fb6d815103d 2024-11-25T12:06:23,120 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/039cd58eb8c34eb092ff18fd5100ed5f to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/039cd58eb8c34eb092ff18fd5100ed5f 2024-11-25T12:06:23,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/e3c269cd715e4f2c8ef137dc04f3c544 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/e3c269cd715e4f2c8ef137dc04f3c544 2024-11-25T12:06:23,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/d326b49a5945424a978572c75765ab0c to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/d326b49a5945424a978572c75765ab0c 2024-11-25T12:06:23,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/4ea0cf819cc8450c8018212b6f73756a to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/4ea0cf819cc8450c8018212b6f73756a 2024-11-25T12:06:23,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/edf8185fb32e4caa9284ff8ebcdbbe82 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/edf8185fb32e4caa9284ff8ebcdbbe82 2024-11-25T12:06:23,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/ef2c25b4fbd34241b642bf871d050063 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/ef2c25b4fbd34241b642bf871d050063 2024-11-25T12:06:23,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/5eca799bdbdb49ea832fc44d1ab12720 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/5eca799bdbdb49ea832fc44d1ab12720 2024-11-25T12:06:23,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/02c6e32794c64200b1ab4d323df8849a to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/02c6e32794c64200b1ab4d323df8849a 2024-11-25T12:06:23,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/bb465d04a92d44e58d653388f78c7c3a to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/bb465d04a92d44e58d653388f78c7c3a 2024-11-25T12:06:23,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/3e2f6f3028de4f089eba67b28a1a11eb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/3e2f6f3028de4f089eba67b28a1a11eb 2024-11-25T12:06:23,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/3dbf24b6fc6e4692bcb6e03f2ff603eb to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/3dbf24b6fc6e4692bcb6e03f2ff603eb 2024-11-25T12:06:23,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/95b906fffbac4349a0e8c116f13b01ac to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/95b906fffbac4349a0e8c116f13b01ac 2024-11-25T12:06:23,148 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/recovered.edits/418.seqid, newMaxSeqId=418, maxSeqId=4 2024-11-25T12:06:23,148 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. 2024-11-25T12:06:23,148 DEBUG [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1676): Region close journal for 23c89a604d5eddf398c6d709d0d8bf0d: Waiting for close lock at 1732536382068Running coprocessor pre-close hooks at 1732536382068Disabling compacts and flushes for region at 1732536382068Disabling writes for close at 1732536382069 (+1 ms)Obtaining lock to block concurrent updates at 1732536382069Preparing flush snapshotting stores in 23c89a604d5eddf398c6d709d0d8bf0d at 1732536382069Finished memstore snapshotting TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732536382069Flushing stores of TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d. at 1732536382070 (+1 ms)Flushing 23c89a604d5eddf398c6d709d0d8bf0d/A: creating writer at 1732536382070Flushing mob file 23c89a604d5eddf398c6d709d0d8bf0d/A: appending metadata at 1732536382081 (+11 ms)Flushing mob file 23c89a604d5eddf398c6d709d0d8bf0d/A: closing flushed file at 1732536382081Flushing 23c89a604d5eddf398c6d709d0d8bf0d/A: appending metadata at 1732536382102 (+21 ms)Flushing 23c89a604d5eddf398c6d709d0d8bf0d/A: closing flushed file at 1732536382102Flushing 23c89a604d5eddf398c6d709d0d8bf0d/B: creating writer at 1732536382514 (+412 ms)Flushing 23c89a604d5eddf398c6d709d0d8bf0d/B: appending metadata at 1732536382517 (+3 ms)Flushing 23c89a604d5eddf398c6d709d0d8bf0d/B: closing flushed file at 1732536382517Flushing 23c89a604d5eddf398c6d709d0d8bf0d/C: creating writer at 1732536382937 (+420 ms)Flushing 23c89a604d5eddf398c6d709d0d8bf0d/C: appending metadata at 1732536382940 (+3 ms)Flushing 23c89a604d5eddf398c6d709d0d8bf0d/C: closing flushed file at 1732536382940Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ad85aa3: reopening flushed file at 1732536382953 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2629245f: reopening flushed file at 1732536382964 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e3c6b63: reopening flushed file at 1732536382975 (+11 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 23c89a604d5eddf398c6d709d0d8bf0d in 911ms, sequenceid=415, compaction requested=true at 1732536382980 (+5 ms)Writing region close event to WAL at 1732536383141 (+161 ms)Running coprocessor post-close hooks at 1732536383148 (+7 ms)Closed at 1732536383148 2024-11-25T12:06:23,150 INFO [RS_CLOSE_REGION-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(157): Closed 23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:23,151 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=158 updating hbase:meta row=23c89a604d5eddf398c6d709d0d8bf0d, regionState=CLOSED 2024-11-25T12:06:23,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=158, state=RUNNABLE, hasLock=false; CloseRegionProcedure 23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 because future has completed 2024-11-25T12:06:23,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-25T12:06:23,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseRegionProcedure 23c89a604d5eddf398c6d709d0d8bf0d, server=2bf237fd86ca,37643,1732536214674 in 1.2370 sec 2024-11-25T12:06:23,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-11-25T12:06:23,156 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=23c89a604d5eddf398c6d709d0d8bf0d, UNASSIGN in 1.2450 sec 2024-11-25T12:06:23,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-25T12:06:23,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.2500 sec 2024-11-25T12:06:23,159 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732536383159"}]},"ts":"1732536383159"} 2024-11-25T12:06:23,160 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-25T12:06:23,160 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-25T12:06:23,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.2610 sec 2024-11-25T12:06:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-25T12:06:24,057 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:06:24,058 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-25T12:06:24,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:06:24,060 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=160, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:06:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T12:06:24,060 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=160, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:06:24,063 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,065 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C, FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/recovered.edits] 2024-11-25T12:06:24,068 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/64565f18aaef4b42a405301fefe37141 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/64565f18aaef4b42a405301fefe37141 2024-11-25T12:06:24,069 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/b905d1b6476e4c1fa81a805f5ea036b6 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/b905d1b6476e4c1fa81a805f5ea036b6 2024-11-25T12:06:24,070 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/de7d5fdcb67241a7b22b8632fce48241 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/de7d5fdcb67241a7b22b8632fce48241 2024-11-25T12:06:24,071 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/f211da3a7240418f9255d4f1b71dcbd5 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/A/f211da3a7240418f9255d4f1b71dcbd5 2024-11-25T12:06:24,073 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/05195521d6ba490fb6b9c864da3dd278 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/05195521d6ba490fb6b9c864da3dd278 2024-11-25T12:06:24,074 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/c6d060684bf94d899b300bb3de0227c3 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/c6d060684bf94d899b300bb3de0227c3 2024-11-25T12:06:24,075 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/d02295e299124b5aad496461e703fcb6 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/d02295e299124b5aad496461e703fcb6 2024-11-25T12:06:24,076 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/e15427103cd24a9e8b8d02828ad07a89 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/B/e15427103cd24a9e8b8d02828ad07a89 2024-11-25T12:06:24,078 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/0573f5ba2feb49f389b0e1fc7555921e to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/0573f5ba2feb49f389b0e1fc7555921e 2024-11-25T12:06:24,079 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/343f6ce4f7c440019cf818243ea17656 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/343f6ce4f7c440019cf818243ea17656 2024-11-25T12:06:24,080 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/8f5b87a1aaab4b34af906c8bbc5a9507 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/8f5b87a1aaab4b34af906c8bbc5a9507 2024-11-25T12:06:24,081 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/fcc51ac59a02461e96497351f9d824b1 to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/C/fcc51ac59a02461e96497351f9d824b1 2024-11-25T12:06:24,083 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/recovered.edits/418.seqid to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d/recovered.edits/418.seqid 2024-11-25T12:06:24,084 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/default/TestAcidGuarantees/23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,084 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-25T12:06:24,084 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T12:06:24,085 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-25T12:06:24,088 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125065eda11d3754862b50849b81cdcb5ee_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125065eda11d3754862b50849b81cdcb5ee_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,089 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112515d3bdeef1224b5b9945a9f25e7d5cc9_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112515d3bdeef1224b5b9945a9f25e7d5cc9_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,090 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112515f3fed3ba9f4e4387d4afbefb8fb0ca_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112515f3fed3ba9f4e4387d4afbefb8fb0ca_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,091 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251b8df6293b8f4a4cadb1d1f26486acdb_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411251b8df6293b8f4a4cadb1d1f26486acdb_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,092 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252c2823c5179d48e9b9494e57fe46959a_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252c2823c5179d48e9b9494e57fe46959a_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,093 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252fd85e0131a542aab2d37a216b1f8c32_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411252fd85e0131a542aab2d37a216b1f8c32_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,094 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112532f77b1a97534300aab941a4a44da106_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112532f77b1a97534300aab941a4a44da106_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,094 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253b6a6142363d41338a146e7f41b8a0f3_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411253b6a6142363d41338a146e7f41b8a0f3_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,096 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112545b8e3e0aa87446e90c8b23f602d668f_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112545b8e3e0aa87446e90c8b23f602d668f_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,096 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112545f8ad00412a4aba81611f8c3bccd4b7_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112545f8ad00412a4aba81611f8c3bccd4b7_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,098 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255633be70e59d4820b547ee956366c49f_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411255633be70e59d4820b547ee956366c49f_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,099 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112584c8244009a340ab890034a8943baeaa_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112584c8244009a340ab890034a8943baeaa_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,100 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258699174038d14ee7bd09801d002653ad_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411258699174038d14ee7bd09801d002653ad_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,101 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ae9ad265f39945c88faf4311584c77b7_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125ae9ad265f39945c88faf4311584c77b7_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,102 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b7c26231f3a5447cb7c3a9ee7e278c36_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125b7c26231f3a5447cb7c3a9ee7e278c36_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,103 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125beb0edce1e494828ab1725e46596e3de_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125beb0edce1e494828ab1725e46596e3de_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,104 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cab45ed80fa742ff971595d9f7a6efce_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125cab45ed80fa742ff971595d9f7a6efce_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,106 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d3442a81b134465c98fb9d5b07967e18_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d3442a81b134465c98fb9d5b07967e18_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,106 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d48862062f624786a11f6820339952e8_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d48862062f624786a11f6820339952e8_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,108 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d4a6f688dc23472aac15540cfc45788e_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125d4a6f688dc23472aac15540cfc45788e_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,109 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125db9b31b6663d4560a71d211b4e257abf_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125db9b31b6663d4560a71d211b4e257abf_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,110 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f4fe1229ba044b1abd1da40f5b154815_23c89a604d5eddf398c6d709d0d8bf0d to hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241125f4fe1229ba044b1abd1da40f5b154815_23c89a604d5eddf398c6d709d0d8bf0d 2024-11-25T12:06:24,110 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-25T12:06:24,113 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=160, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:06:24,116 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-25T12:06:24,118 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-25T12:06:24,119 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=160, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:06:24,119 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-25T12:06:24,119 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732536384119"}]},"ts":"9223372036854775807"} 2024-11-25T12:06:24,121 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-25T12:06:24,121 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 23c89a604d5eddf398c6d709d0d8bf0d, NAME => 'TestAcidGuarantees,,1732536354327.23c89a604d5eddf398c6d709d0d8bf0d.', STARTKEY => '', ENDKEY => ''}] 2024-11-25T12:06:24,121 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-25T12:06:24,121 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732536384121"}]},"ts":"9223372036854775807"} 2024-11-25T12:06:24,123 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-25T12:06:24,123 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=160, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-25T12:06:24,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 66 msec 2024-11-25T12:06:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-25T12:06:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-25T12:06:24,171 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-25T12:06:24,180 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=239 (was 242), OpenFileDescriptor=449 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1193 (was 1363), ProcessCount=11 (was 11), AvailableMemoryMB=7350 (was 5751) - AvailableMemoryMB LEAK? - 2024-11-25T12:06:24,180 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T12:06:24,181 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T12:06:24,181 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:06:24,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:24,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:24,181 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T12:06:24,181 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T12:06:24,181 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1519452659, stopped=false 2024-11-25T12:06:24,182 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2bf237fd86ca,40019,1732536213609 2024-11-25T12:06:24,254 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-25T12:06:24,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T12:06:24,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:06:24,270 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T12:06:24,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T12:06:24,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:06:24,271 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T12:06:24,271 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:06:24,271 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:24,271 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2bf237fd86ca,37643,1732536214674' ***** 2024-11-25T12:06:24,271 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T12:06:24,271 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T12:06:24,271 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T12:06:24,272 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T12:06:24,273 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T12:06:24,274 INFO [RS:0;2bf237fd86ca:37643 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T12:06:24,274 INFO [RS:0;2bf237fd86ca:37643 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T12:06:24,274 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(959): stopping server 2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:24,274 INFO [RS:0;2bf237fd86ca:37643 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T12:06:24,274 INFO [RS:0;2bf237fd86ca:37643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2bf237fd86ca:37643. 2024-11-25T12:06:24,274 DEBUG [RS:0;2bf237fd86ca:37643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T12:06:24,274 DEBUG [RS:0;2bf237fd86ca:37643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:24,274 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T12:06:24,274 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T12:06:24,274 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T12:06:24,275 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T12:06:24,277 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-25T12:06:24,277 DEBUG [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-25T12:06:24,277 DEBUG [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T12:06:24,277 DEBUG [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T12:06:24,277 INFO [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T12:06:24,277 DEBUG [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T12:06:24,277 DEBUG [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T12:06:24,277 DEBUG [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T12:06:24,277 INFO [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-25T12:06:24,284 INFO [regionserver/2bf237fd86ca:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T12:06:24,323 DEBUG [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/.tmp/info/a8205f06c7254bcca775e245d78ece0c is 95, key is TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69./info:/1732536245244/DeleteFamily/seqid=0 2024-11-25T12:06:24,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742468_1644 (size=6514) 2024-11-25T12:06:24,477 DEBUG [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T12:06:24,677 DEBUG [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T12:06:24,750 INFO [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/.tmp/info/a8205f06c7254bcca775e245d78ece0c 2024-11-25T12:06:24,776 DEBUG [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/.tmp/ns/ea03bd19179b4bc489ee8004ef4253b2 is 93, key is TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69./ns:/1732536245178/DeleteFamily/seqid=0 2024-11-25T12:06:24,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742469_1645 (size=5906) 2024-11-25T12:06:24,878 DEBUG [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T12:06:24,893 INFO [regionserver/2bf237fd86ca:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-25T12:06:24,893 INFO [regionserver/2bf237fd86ca:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-25T12:06:25,078 DEBUG [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T12:06:25,185 INFO [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/.tmp/ns/ea03bd19179b4bc489ee8004ef4253b2 2024-11-25T12:06:25,209 DEBUG [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/.tmp/rep_barrier/b69b1b0b12a349688b7b4ca21dda0395 is 102, key is TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69./rep_barrier:/1732536245178/DeleteFamily/seqid=0 2024-11-25T12:06:25,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742470_1646 (size=6025) 2024-11-25T12:06:25,278 DEBUG [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T12:06:25,478 DEBUG [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T12:06:25,614 INFO [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/.tmp/rep_barrier/b69b1b0b12a349688b7b4ca21dda0395 2024-11-25T12:06:25,638 DEBUG [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/.tmp/table/531a89f0121c46bbba1b28823a92fd64 is 96, key is TestAcidGuarantees,,1732536219050.057d2058e8441af178fff3bc63c12b69./table:/1732536245178/DeleteFamily/seqid=0 2024-11-25T12:06:25,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742471_1647 (size=5936) 2024-11-25T12:06:25,642 INFO [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/.tmp/table/531a89f0121c46bbba1b28823a92fd64 2024-11-25T12:06:25,646 DEBUG [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/.tmp/info/a8205f06c7254bcca775e245d78ece0c as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/info/a8205f06c7254bcca775e245d78ece0c 2024-11-25T12:06:25,649 INFO [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/info/a8205f06c7254bcca775e245d78ece0c, entries=12, sequenceid=90, filesize=6.4 K 2024-11-25T12:06:25,650 DEBUG [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/.tmp/ns/ea03bd19179b4bc489ee8004ef4253b2 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/ns/ea03bd19179b4bc489ee8004ef4253b2 2024-11-25T12:06:25,653 INFO [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/ns/ea03bd19179b4bc489ee8004ef4253b2, entries=8, sequenceid=90, filesize=5.8 K 2024-11-25T12:06:25,654 DEBUG [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/.tmp/rep_barrier/b69b1b0b12a349688b7b4ca21dda0395 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/rep_barrier/b69b1b0b12a349688b7b4ca21dda0395 2024-11-25T12:06:25,657 INFO [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/rep_barrier/b69b1b0b12a349688b7b4ca21dda0395, entries=6, sequenceid=90, filesize=5.9 K 2024-11-25T12:06:25,657 DEBUG [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/.tmp/table/531a89f0121c46bbba1b28823a92fd64 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/table/531a89f0121c46bbba1b28823a92fd64 2024-11-25T12:06:25,661 INFO [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/table/531a89f0121c46bbba1b28823a92fd64, entries=7, sequenceid=90, filesize=5.8 K 2024-11-25T12:06:25,661 INFO [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1384ms, sequenceid=90, compaction requested=false 2024-11-25T12:06:25,676 DEBUG [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-25T12:06:25,677 DEBUG [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T12:06:25,677 INFO [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T12:06:25,677 DEBUG [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732536384277Running coprocessor pre-close hooks at 1732536384277Disabling compacts and flushes for region at 1732536384277Disabling writes for close at 1732536384277Obtaining lock to block concurrent updates at 1732536384277Preparing flush snapshotting stores in 1588230740 at 1732536384277Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732536384278 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732536384278Flushing 1588230740/info: creating writer at 1732536384279 (+1 ms)Flushing 1588230740/info: appending metadata at 1732536384323 (+44 ms)Flushing 1588230740/info: closing flushed file at 1732536384323Flushing 1588230740/ns: creating writer at 1732536384754 (+431 ms)Flushing 1588230740/ns: appending metadata at 1732536384775 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1732536384775Flushing 1588230740/rep_barrier: creating writer at 1732536385189 (+414 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732536385209 (+20 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732536385209Flushing 1588230740/table: creating writer at 1732536385617 (+408 ms)Flushing 1588230740/table: appending metadata at 1732536385637 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732536385637Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fa8b452: reopening flushed file at 1732536385645 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@437fcb76: reopening flushed file at 1732536385649 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53bd5b03: reopening flushed file at 1732536385653 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4528a24a: reopening flushed file at 1732536385657 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1384ms, sequenceid=90, compaction requested=false at 1732536385661 (+4 ms)Writing region close event to WAL at 1732536385669 (+8 ms)Running coprocessor post-close hooks at 1732536385677 (+8 ms)Closed at 1732536385677 2024-11-25T12:06:25,677 DEBUG [RS_CLOSE_META-regionserver/2bf237fd86ca:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T12:06:25,679 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(976): stopping server 2bf237fd86ca,37643,1732536214674; all regions closed. 2024-11-25T12:06:25,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741834_1010 (size=25344) 2024-11-25T12:06:25,686 DEBUG [RS:0;2bf237fd86ca:37643 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/oldWALs 2024-11-25T12:06:25,686 INFO [RS:0;2bf237fd86ca:37643 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2bf237fd86ca%2C37643%2C1732536214674.meta:.meta(num 1732536218472) 2024-11-25T12:06:25,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741833_1009 (size=14782038) 2024-11-25T12:06:25,693 DEBUG [RS:0;2bf237fd86ca:37643 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/oldWALs 2024-11-25T12:06:25,693 INFO [RS:0;2bf237fd86ca:37643 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2bf237fd86ca%2C37643%2C1732536214674:(num 1732536217992) 2024-11-25T12:06:25,693 DEBUG [RS:0;2bf237fd86ca:37643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T12:06:25,693 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T12:06:25,693 INFO [RS:0;2bf237fd86ca:37643 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T12:06:25,693 INFO [RS:0;2bf237fd86ca:37643 {}] hbase.ChoreService(370): Chore service for: regionserver/2bf237fd86ca:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T12:06:25,694 INFO [RS:0;2bf237fd86ca:37643 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T12:06:25,694 INFO [regionserver/2bf237fd86ca:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T12:06:25,694 INFO [RS:0;2bf237fd86ca:37643 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37643 2024-11-25T12:06:25,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2bf237fd86ca,37643,1732536214674 2024-11-25T12:06:25,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T12:06:25,700 INFO [RS:0;2bf237fd86ca:37643 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T12:06:25,701 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2bf237fd86ca,37643,1732536214674] 2024-11-25T12:06:25,713 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2bf237fd86ca,37643,1732536214674 already deleted, retry=false 2024-11-25T12:06:25,713 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2bf237fd86ca,37643,1732536214674 expired; onlineServers=0 2024-11-25T12:06:25,713 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2bf237fd86ca,40019,1732536213609' ***** 2024-11-25T12:06:25,713 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T12:06:25,713 INFO [M:0;2bf237fd86ca:40019 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T12:06:25,714 INFO [M:0;2bf237fd86ca:40019 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T12:06:25,714 DEBUG [M:0;2bf237fd86ca:40019 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T12:06:25,714 DEBUG [M:0;2bf237fd86ca:40019 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T12:06:25,714 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T12:06:25,714 INFO [M:0;2bf237fd86ca:40019 {}] hbase.ChoreService(370): Chore service for: master/2bf237fd86ca:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T12:06:25,714 INFO [M:0;2bf237fd86ca:40019 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T12:06:25,714 DEBUG [M:0;2bf237fd86ca:40019 {}] master.HMaster(1795): Stopping service threads 2024-11-25T12:06:25,714 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster-HFileCleaner.small.0-1732536217564 {}] cleaner.HFileCleaner(306): Exit Thread[master/2bf237fd86ca:0:becomeActiveMaster-HFileCleaner.small.0-1732536217564,5,FailOnTimeoutGroup] 2024-11-25T12:06:25,714 INFO [M:0;2bf237fd86ca:40019 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T12:06:25,714 INFO [M:0;2bf237fd86ca:40019 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T12:06:25,714 DEBUG [master/2bf237fd86ca:0:becomeActiveMaster-HFileCleaner.large.0-1732536217563 {}] cleaner.HFileCleaner(306): Exit Thread[master/2bf237fd86ca:0:becomeActiveMaster-HFileCleaner.large.0-1732536217563,5,FailOnTimeoutGroup] 2024-11-25T12:06:25,715 ERROR [M:0;2bf237fd86ca:40019 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (745627032) connection to localhost/127.0.0.1:33743 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:33743,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-25T12:06:25,715 INFO [M:0;2bf237fd86ca:40019 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T12:06:25,718 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T12:06:25,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T12:06:25,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T12:06:25,725 DEBUG [M:0;2bf237fd86ca:40019 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-25T12:06:25,725 DEBUG [M:0;2bf237fd86ca:40019 {}] master.ActiveMasterManager(353): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-25T12:06:25,725 INFO [M:0;2bf237fd86ca:40019 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/.lastflushedseqids 2024-11-25T12:06:25,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742472_1648 (size=72) 2024-11-25T12:06:25,742 INFO [M:0;2bf237fd86ca:40019 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T12:06:25,742 INFO [M:0;2bf237fd86ca:40019 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T12:06:25,742 DEBUG [M:0;2bf237fd86ca:40019 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T12:06:25,742 INFO [M:0;2bf237fd86ca:40019 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:06:25,742 DEBUG [M:0;2bf237fd86ca:40019 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:06:25,742 DEBUG [M:0;2bf237fd86ca:40019 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T12:06:25,742 DEBUG [M:0;2bf237fd86ca:40019 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:06:25,742 INFO [M:0;2bf237fd86ca:40019 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=760.83 KB heapSize=935.72 KB 2024-11-25T12:06:25,765 DEBUG [M:0;2bf237fd86ca:40019 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/38eee94a20a94b77a2847c60603df50d is 82, key is hbase:meta,,1/info:regioninfo/1732536218609/Put/seqid=0 2024-11-25T12:06:25,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742473_1649 (size=5672) 2024-11-25T12:06:25,774 INFO [M:0;2bf237fd86ca:40019 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2169 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/38eee94a20a94b77a2847c60603df50d 2024-11-25T12:06:25,805 INFO [RS:0;2bf237fd86ca:37643 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T12:06:25,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T12:06:25,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37643-0x10076d2ae840001, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T12:06:25,806 INFO [RS:0;2bf237fd86ca:37643 {}] regionserver.HRegionServer(1031): Exiting; stopping=2bf237fd86ca,37643,1732536214674; zookeeper connection closed. 2024-11-25T12:06:25,813 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7c791190 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7c791190 2024-11-25T12:06:25,813 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-25T12:06:25,823 DEBUG [M:0;2bf237fd86ca:40019 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0c5adba7163d4e2ba598a9f9c114f974 is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x89/proc:d/1732536356009/Put/seqid=0 2024-11-25T12:06:25,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742474_1650 (size=42454) 2024-11-25T12:06:25,884 INFO [M:0;2bf237fd86ca:40019 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=760.27 KB at sequenceid=2169 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0c5adba7163d4e2ba598a9f9c114f974 2024-11-25T12:06:25,912 INFO [M:0;2bf237fd86ca:40019 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0c5adba7163d4e2ba598a9f9c114f974 2024-11-25T12:06:25,950 DEBUG [M:0;2bf237fd86ca:40019 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/03fed761701b40e5965b6335a45b6617 is 69, key is 2bf237fd86ca,37643,1732536214674/rs:state/1732536217684/Put/seqid=0 2024-11-25T12:06:25,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073742475_1651 (size=5156) 2024-11-25T12:06:26,358 INFO [M:0;2bf237fd86ca:40019 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2169 (bloomFilter=true), to=hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/03fed761701b40e5965b6335a45b6617 2024-11-25T12:06:26,362 DEBUG [M:0;2bf237fd86ca:40019 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/38eee94a20a94b77a2847c60603df50d as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/38eee94a20a94b77a2847c60603df50d 2024-11-25T12:06:26,365 INFO [M:0;2bf237fd86ca:40019 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/38eee94a20a94b77a2847c60603df50d, entries=8, sequenceid=2169, filesize=5.5 K 2024-11-25T12:06:26,366 DEBUG [M:0;2bf237fd86ca:40019 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0c5adba7163d4e2ba598a9f9c114f974 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0c5adba7163d4e2ba598a9f9c114f974 2024-11-25T12:06:26,369 INFO [M:0;2bf237fd86ca:40019 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0c5adba7163d4e2ba598a9f9c114f974 2024-11-25T12:06:26,369 INFO [M:0;2bf237fd86ca:40019 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0c5adba7163d4e2ba598a9f9c114f974, entries=160, sequenceid=2169, filesize=41.5 K 2024-11-25T12:06:26,369 DEBUG [M:0;2bf237fd86ca:40019 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/03fed761701b40e5965b6335a45b6617 as hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/03fed761701b40e5965b6335a45b6617 2024-11-25T12:06:26,372 INFO [M:0;2bf237fd86ca:40019 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33743/user/jenkins/test-data/216edf48-d796-a5cb-ed6a-6486091c3b3d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/03fed761701b40e5965b6335a45b6617, entries=1, sequenceid=2169, filesize=5.0 K 2024-11-25T12:06:26,373 INFO [M:0;2bf237fd86ca:40019 {}] regionserver.HRegion(3140): Finished flush of dataSize ~760.83 KB/779090, heapSize ~935.42 KB/957872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 631ms, sequenceid=2169, compaction requested=false 2024-11-25T12:06:26,374 INFO [M:0;2bf237fd86ca:40019 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T12:06:26,374 DEBUG [M:0;2bf237fd86ca:40019 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732536385742Disabling compacts and flushes for region at 1732536385742Disabling writes for close at 1732536385742Obtaining lock to block concurrent updates at 1732536385742Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732536385742Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=779090, getHeapSize=958112, getOffHeapSize=0, getCellsCount=2354 at 1732536385743 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732536385743Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732536385743Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732536385764 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732536385764Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732536385778 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732536385822 (+44 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732536385822Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732536385913 (+91 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732536385949 (+36 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732536385949Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79db6c3e: reopening flushed file at 1732536386362 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7631e479: reopening flushed file at 1732536386365 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@196b14a9: reopening flushed file at 1732536386369 (+4 ms)Finished flush of dataSize ~760.83 KB/779090, heapSize ~935.42 KB/957872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 631ms, sequenceid=2169, compaction requested=false at 1732536386373 (+4 ms)Writing region close event to WAL at 1732536386374 (+1 ms)Closed at 1732536386374 2024-11-25T12:06:26,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36647 is added to blk_1073741830_1006 (size=921234) 2024-11-25T12:06:26,377 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T12:06:26,378 INFO [M:0;2bf237fd86ca:40019 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T12:06:26,378 INFO [M:0;2bf237fd86ca:40019 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40019 2024-11-25T12:06:26,378 INFO [M:0;2bf237fd86ca:40019 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T12:06:26,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T12:06:26,480 INFO [M:0;2bf237fd86ca:40019 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T12:06:26,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40019-0x10076d2ae840000, quorum=127.0.0.1:63609, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T12:06:26,490 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1e531827{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T12:06:26,492 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4ea205b6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T12:06:26,492 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T12:06:26,492 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@24457f8e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T12:06:26,493 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@350067e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/hadoop.log.dir/,STOPPED} 2024-11-25T12:06:26,496 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T12:06:26,497 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T12:06:26,497 WARN [BP-261260195-172.17.0.3-1732536209581 heartbeating to localhost/127.0.0.1:33743 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T12:06:26,497 WARN [BP-261260195-172.17.0.3-1732536209581 heartbeating to localhost/127.0.0.1:33743 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-261260195-172.17.0.3-1732536209581 (Datanode Uuid 77d49c2a-8a68-48a8-be79-2fca156f8fc0) service to localhost/127.0.0.1:33743 2024-11-25T12:06:26,500 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/cluster_2876dd1b-38a2-aa39-0643-f2db42ed69c6/data/data1/current/BP-261260195-172.17.0.3-1732536209581 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T12:06:26,500 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/cluster_2876dd1b-38a2-aa39-0643-f2db42ed69c6/data/data2/current/BP-261260195-172.17.0.3-1732536209581 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T12:06:26,501 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T12:06:26,509 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7f405bdd{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T12:06:26,513 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5b829fcb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T12:06:26,513 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T12:06:26,513 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31a6a643{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T12:06:26,513 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@610d6b27{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0933374b-d423-b723-4883-b3e039764072/hadoop.log.dir/,STOPPED} 2024-11-25T12:06:26,536 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T12:06:26,679 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down